summaryrefslogtreecommitdiff
path: root/ishtar_common
diff options
context:
space:
mode:
Diffstat (limited to 'ishtar_common')
-rw-r--r--ishtar_common/admin.py94
-rw-r--r--ishtar_common/data_importer.py7
-rw-r--r--ishtar_common/management/commands/update_search_vectors.py24
-rw-r--r--ishtar_common/migrations/0015_auto_20171011_1644.py36
-rw-r--r--ishtar_common/migrations/0016_auto_20171016_1104.py30
-rw-r--r--ishtar_common/migrations/0017_auto_20171016_1320.py29
-rw-r--r--ishtar_common/migrations/0018_auto_20171017_1840.py72
-rw-r--r--ishtar_common/models.py205
-rw-r--r--ishtar_common/templates/ishtar/blocks/sheet_json.html11
-rw-r--r--ishtar_common/tests.py45
-rw-r--r--ishtar_common/utils.py44
-rw-r--r--ishtar_common/views.py26
-rw-r--r--ishtar_common/wizards.py3
13 files changed, 606 insertions, 20 deletions
diff --git a/ishtar_common/admin.py b/ishtar_common/admin.py
index cec61a51e..2df910ffd 100644
--- a/ishtar_common/admin.py
+++ b/ishtar_common/admin.py
@@ -20,11 +20,14 @@
import csv
from ajax_select import make_ajax_form
+from ajax_select.fields import AutoCompleteSelectField, \
+ AutoCompleteSelectMultipleField
from django.conf import settings
from django.contrib import admin
from django.contrib.auth.admin import GroupAdmin, UserAdmin
from django.contrib.auth.models import Group, User
+from django.contrib.contenttypes.models import ContentType
from django.contrib.sites.admin import SiteAdmin
from django.contrib.sites.models import Site
from django.contrib.gis.forms import PointField, OSMWidget
@@ -114,12 +117,22 @@ def export_as_csv_action(description=_(u"Export selected as CSV file"),
class HistorizedObjectAdmin(admin.ModelAdmin):
- readonly_fields = ['history_creator', 'history_modifier',]
+ readonly_fields = ['history_creator', 'history_modifier', 'search_vector']
def save_model(self, request, obj, form, change):
obj.history_modifier = request.user
obj.save()
+ def get_readonly_fields(self, request, obj=None):
+ if obj: # editing an existing object
+ return tuple(self.readonly_fields or []) + tuple(['imports'])
+ return self.readonly_fields
+
+ def get_exclude(self, request, obj=None):
+ if not obj:
+ return tuple(self.exclude or []) + tuple(['imports'])
+ return self.exclude
+
class MyGroupAdmin(GroupAdmin):
class Media:
@@ -153,7 +166,6 @@ class OrganizationAdmin(HistorizedObjectAdmin):
list_filter = ("organization_type",)
search_fields = ('name',)
exclude = ('merge_key', 'merge_exclusion', 'merge_candidate', )
- readonly_fields = HistorizedObjectAdmin.readonly_fields + ['imports']
model = models.Organization
admin_site.register(models.Organization, OrganizationAdmin)
@@ -164,31 +176,51 @@ class PersonAdmin(HistorizedObjectAdmin):
list_filter = ("person_types",)
search_fields = ('name', 'surname', 'email', 'raw_name')
exclude = ('merge_key', 'merge_exclusion', 'merge_candidate', )
- readonly_fields = HistorizedObjectAdmin.readonly_fields + ['imports']
form = make_ajax_form(models.Person, {'attached_to': 'organization'})
model = models.Person
admin_site.register(models.Person, PersonAdmin)
+class AdminRelatedTownForm(forms.ModelForm):
+ class Meta:
+ model = models.Town.children.through
+ exclude = []
+ from_town = AutoCompleteSelectField(
+ 'town', required=True, label=_(u"Parent"))
+
+
class AdminTownForm(forms.ModelForm):
class Meta:
model = models.Town
- exclude = []
+ exclude = ['imports']
center = PointField(label=_(u"center"), required=False,
widget=OSMWidget)
+ children = AutoCompleteSelectMultipleField('town', required=False,
+ label=_(u"Town children"))
+
+
+class TownParentInline(admin.TabularInline):
+ model = models.Town.children.through
+ fk_name = 'to_town'
+ form = AdminRelatedTownForm
+ verbose_name = _(u"Parent")
+ verbose_name_plural = _(u"Parents")
+ extra = 1
class TownAdmin(admin.ModelAdmin):
+ model = models.Town
list_display = ['name', ]
search_fields = ['name']
+ readonly_fields = ['cached_label']
if settings.COUNTRY == 'fr':
list_display += ['numero_insee', 'departement', ]
search_fields += ['numero_insee', 'departement__label', ]
list_filter = ("departement",)
- readonly_fields = ['imports']
- model = models.Town
form = AdminTownForm
+ inlines = [TownParentInline]
+
admin_site.register(models.Town, TownAdmin)
@@ -333,6 +365,56 @@ class ItemKeyAdmin(admin.ModelAdmin):
admin_site.register(models.ItemKey, ItemKeyAdmin)
+class JsonContentTypeFormMixin(object):
+ class Meta:
+ model = models.JsonDataSection
+ exclude = []
+
+ def __init__(self, *args, **kwargs):
+ super(JsonContentTypeFormMixin, self).__init__(*args, **kwargs)
+ choices = []
+ for pk, label in self.fields['content_type'].choices:
+ if not pk:
+ choices.append((pk, label))
+ continue
+ ct = ContentType.objects.get(pk=pk)
+ model_class = ct.model_class()
+ if hasattr(model_class, 'data') and \
+ not hasattr(model_class, 'history_type'):
+ choices.append((pk, label))
+ self.fields['content_type'].choices = sorted(choices,
+ key=lambda x: x[1])
+
+
+class JsonDataSectionForm(JsonContentTypeFormMixin, forms.ModelForm):
+ class Meta:
+ model = models.JsonDataSection
+ exclude = []
+
+
+class JsonDataSectionAdmin(admin.ModelAdmin):
+ list_display = ['name', 'content_type', 'order']
+ form = JsonDataSectionForm
+
+
+admin_site.register(models.JsonDataSection, JsonDataSectionAdmin)
+
+
+class JsonDataFieldForm(JsonContentTypeFormMixin, forms.ModelForm):
+ class Meta:
+ model = models.JsonDataField
+ exclude = []
+
+
+class JsonDataFieldAdmin(admin.ModelAdmin):
+ list_display = ['name', 'content_type', 'key', 'display',
+ 'order', 'section']
+ form = JsonDataFieldForm
+
+
+admin_site.register(models.JsonDataField, JsonDataFieldAdmin)
+
+
class AdministrationScriptAdmin(admin.ModelAdmin):
list_display = ['name', 'path']
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index 9caebb2dd..e8ec43ab2 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -1486,6 +1486,9 @@ class Importer(object):
# importer trigger
self._set_importer_trigger(cls, attribute, data)
return
+ if attribute == 'data': # json field
+ # no need to do anything
+ return
try:
field_object = cls._meta.get_field(attribute)
except FieldDoesNotExist:
@@ -1570,8 +1573,8 @@ class Importer(object):
create_dict = copy.deepcopy(data)
for k in create_dict.keys():
- # filter unnecessary default values
- if type(create_dict[k]) == dict:
+ # filter unnecessary default values but not the json field
+ if type(create_dict[k]) == dict and k != 'data':
create_dict.pop(k)
# File doesn't like deepcopy
elif type(create_dict[k]) == File:
diff --git a/ishtar_common/management/commands/update_search_vectors.py b/ishtar_common/management/commands/update_search_vectors.py
new file mode 100644
index 000000000..c73a6e88e
--- /dev/null
+++ b/ishtar_common/management/commands/update_search_vectors.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import sys
+
+from django.core.management.base import BaseCommand
+import django.apps
+
+
+class Command(BaseCommand):
+ help = "./manage.py update_search_vectors\n\n"\
+ "Update full texte search vectors."
+
+ def handle(self, *args, **options):
+ for model in django.apps.apps.get_models():
+ if hasattr(model, "update_search_vector") and \
+ getattr(model, "BASE_SEARCH_VECTORS", None):
+ self.stdout.write("\n* update {}".format(model))
+ total = model.objects.count()
+ for idx, item in enumerate(model.objects.all()):
+ sys.stdout.write("\r{}/{} ".format(idx, total))
+ sys.stdout.flush()
+ item.update_search_vector()
+ self.stdout.write("\n")
diff --git a/ishtar_common/migrations/0015_auto_20171011_1644.py b/ishtar_common/migrations/0015_auto_20171011_1644.py
new file mode 100644
index 000000000..a9f4499c2
--- /dev/null
+++ b/ishtar_common/migrations/0015_auto_20171011_1644.py
@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11 on 2017-10-11 16:44
+from __future__ import unicode_literals
+
+import django.contrib.postgres.search
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('ishtar_common', '0014_ishtarsiteprofile_preservation'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='historicalorganization',
+ name='search_vector',
+ field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'),
+ ),
+ migrations.AddField(
+ model_name='historicalperson',
+ name='search_vector',
+ field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'),
+ ),
+ migrations.AddField(
+ model_name='organization',
+ name='search_vector',
+ field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'),
+ ),
+ migrations.AddField(
+ model_name='person',
+ name='search_vector',
+ field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'),
+ ),
+ ]
diff --git a/ishtar_common/migrations/0016_auto_20171016_1104.py b/ishtar_common/migrations/0016_auto_20171016_1104.py
new file mode 100644
index 000000000..1d9209bdd
--- /dev/null
+++ b/ishtar_common/migrations/0016_auto_20171016_1104.py
@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11 on 2017-10-16 11:04
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('ishtar_common', '0015_auto_20171011_1644'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='town',
+ name='cached_label',
+ field=models.CharField(blank=True, db_index=True, max_length=500, null=True, verbose_name='Cached name'),
+ ),
+ migrations.AddField(
+ model_name='town',
+ name='children',
+ field=models.ManyToManyField(blank=True, related_name='parents', to='ishtar_common.Town', verbose_name='Town children'),
+ ),
+ migrations.AddField(
+ model_name='town',
+ name='year',
+ field=models.IntegerField(blank=True, help_text='If not filled considered as the older town known.', null=True, verbose_name='Year of creation'),
+ ),
+ ]
diff --git a/ishtar_common/migrations/0017_auto_20171016_1320.py b/ishtar_common/migrations/0017_auto_20171016_1320.py
new file mode 100644
index 000000000..a48b36ce7
--- /dev/null
+++ b/ishtar_common/migrations/0017_auto_20171016_1320.py
@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11 on 2017-10-16 13:20
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('ishtar_common', '0016_auto_20171016_1104'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='town',
+ name='numero_insee',
+ field=models.CharField(max_length=6, verbose_name='Num\xe9ro INSEE'),
+ ),
+ migrations.AlterField(
+ model_name='town',
+ name='year',
+ field=models.IntegerField(blank=True, help_text='Filling this field is relevant to distinguish old towns to new towns.', null=True, verbose_name='Year of creation'),
+ ),
+ migrations.AlterUniqueTogether(
+ name='town',
+ unique_together=set([('numero_insee', 'year')]),
+ ),
+ ]
diff --git a/ishtar_common/migrations/0018_auto_20171017_1840.py b/ishtar_common/migrations/0018_auto_20171017_1840.py
new file mode 100644
index 000000000..0c617a3d5
--- /dev/null
+++ b/ishtar_common/migrations/0018_auto_20171017_1840.py
@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11 on 2017-10-17 18:40
+from __future__ import unicode_literals
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('contenttypes', '0002_remove_content_type_name'),
+ ('ishtar_common', '0017_auto_20171016_1320'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='JsonDataField',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('name', models.CharField(max_length=200, verbose_name='Name')),
+ ('key', models.CharField(help_text='Value of the key in the JSON schema. For hierarchical key use "__" to explain it. For instance the key \'my_subkey\' with data such as {\'my_key\': {\'my_subkey\': \'value\'}} will be reached with my_key__my_subkey.', max_length=200, verbose_name='Key')),
+ ('display', models.BooleanField(default=True, verbose_name='Display')),
+ ('order', models.IntegerField(default=10, verbose_name='Order')),
+ ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')),
+ ],
+ options={
+ 'verbose_name': 'Json data - Field',
+ 'verbose_name_plural': 'Json data - Fields',
+ },
+ ),
+ migrations.CreateModel(
+ name='JsonDataSection',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('name', models.CharField(max_length=200, verbose_name='Name')),
+ ('order', models.IntegerField(default=10, verbose_name='Order')),
+ ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')),
+ ],
+ options={
+ 'ordering': ['name'],
+ 'verbose_name': 'Json data - Menu',
+ 'verbose_name_plural': 'Json data - Menus',
+ },
+ ),
+ migrations.AddField(
+ model_name='historicalorganization',
+ name='data',
+ field=django.contrib.postgres.fields.jsonb.JSONField(db_index=True, default={}),
+ ),
+ migrations.AddField(
+ model_name='historicalperson',
+ name='data',
+ field=django.contrib.postgres.fields.jsonb.JSONField(db_index=True, default={}),
+ ),
+ migrations.AddField(
+ model_name='organization',
+ name='data',
+ field=django.contrib.postgres.fields.jsonb.JSONField(db_index=True, default={}),
+ ),
+ migrations.AddField(
+ model_name='person',
+ name='data',
+ field=django.contrib.postgres.fields.jsonb.JSONField(db_index=True, default={}),
+ ),
+ migrations.AddField(
+ model_name='jsondatafield',
+ name='section',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.JsonDataSection'),
+ ),
+ ]
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index 28a24115b..c3ba4fdd0 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -35,6 +35,8 @@ import tempfile
import time
from django.conf import settings
+from django.contrib.postgres.fields import JSONField
+from django.contrib.postgres.search import SearchVectorField, SearchVector
from django.core.cache import cache
from django.core.exceptions import ObjectDoesNotExist, ValidationError
from django.core.files.uploadedfile import SimpleUploadedFile
@@ -58,7 +60,7 @@ from simple_history.models import HistoricalRecords as BaseHistoricalRecords
from ishtar_common.model_merging import merge_model_objects
from ishtar_common.utils import get_cache, disable_for_loaddata, create_slug,\
- get_all_field_names
+ get_all_field_names, merge_tsvectors, cached_label_changed
from ishtar_common.models_imports import ImporterModel, ImporterType, \
ImporterDefault, ImporterDefaultValues, ImporterColumn, \
@@ -908,6 +910,96 @@ class BulkUpdatedItem(object):
return transaction_id, False
+class JsonDataSection(models.Model):
+ content_type = models.ForeignKey(ContentType)
+ name = models.CharField(_(u"Name"), max_length=200)
+ order = models.IntegerField(_(u"Order"), default=10)
+
+ class Meta:
+ verbose_name = _(u"Json data - Menu")
+ verbose_name_plural = _(u"Json data - Menus")
+ ordering = ['order', 'name']
+
+ def __unicode__(self):
+ return u"{} - {}".format(self.content_type, self.name)
+
+
+class JsonDataField(models.Model):
+ name = models.CharField(_(u"Name"), max_length=200)
+ content_type = models.ForeignKey(ContentType)
+ key = models.CharField(
+ _(u"Key"), max_length=200,
+ help_text=_(u"Value of the key in the JSON schema. For hierarchical "
+ u"key use \"__\" to explain it. For instance the key "
+ u"'my_subkey' with data such as {'my_key': {'my_subkey': "
+ u"'value'}} will be reached with my_key__my_subkey."))
+ display = models.BooleanField(_(u"Display"), default=True)
+ order = models.IntegerField(_(u"Order"), default=10)
+ section = models.ForeignKey(JsonDataSection, blank=True, null=True)
+
+ class Meta:
+ verbose_name = _(u"Json data - Field")
+ verbose_name_plural = _(u"Json data - Fields")
+ ordering = ['order', 'name']
+
+ def __unicode__(self):
+ return u"{} - {}".format(self.content_type, self.name)
+
+ def clean(self):
+ if not self.section:
+ return
+ if self.section.content_type != self.content_type:
+ raise ValidationError(
+ _(u"Content type of the field and of the menu do not match"))
+
+
+class JsonData(models.Model):
+ data = JSONField(default={}, db_index=True, blank=True)
+
+ class Meta:
+ abstract = True
+
+ def pre_save(self):
+ if not self.data:
+ self.data = {}
+
+ @property
+ def json_sections(self):
+ sections = []
+ try:
+ content_type = ContentType.objects.get_for_model(self)
+ except ContentType.DoesNotExists:
+ return sections
+ fields = list(JsonDataField.objects.filter(
+ content_type=content_type, display=True, section__isnull=True
+ ).all()) # no section fields
+
+ fields += list(JsonDataField.objects.filter(
+ content_type=content_type, display=True, section__isnull=False
+ ).order_by('section__order', 'order').all())
+
+ for field in fields:
+ value = None
+ data = self.data.copy()
+ for key in field.key.split('__'):
+ if key in data:
+ value = copy.copy(data[key])
+ data = data[key]
+ else:
+ value = None
+ break
+ if not value:
+ continue
+ if type(value) in (list, tuple):
+ value = u" ; ".join([unicode(v) for v in value])
+ section_name = field.section.name if field.section else None
+ if not sections or section_name != sections[-1][0]:
+ # if section name is identical it is the same
+ sections.append((section_name, []))
+ sections[-1][1].append((field.name, value))
+ return sections
+
+
class Imported(models.Model):
imports = models.ManyToManyField(
Import, blank=True,
@@ -917,9 +1009,85 @@ class Imported(models.Model):
abstract = True
-class BaseHistorizedItem(Imported):
+class FullSearch(models.Model):
+ search_vector = SearchVectorField(_("Search vector"), blank=True, null=True,
+ help_text=_("Auto filled at save"))
+ BASE_SEARCH_VECTORS = []
+ INT_SEARCH_VECTORS = []
+ M2M_SEARCH_VECTORS = []
+ PARENT_SEARCH_VECTORS = []
+
+ class Meta:
+ abstract = True
+
+ def update_search_vector(self, save=True):
+ """
+ Update the search vector
+ :param save: True if you want to save the object immediately
+ :return: True if modified
+ """
+ if not self.BASE_SEARCH_VECTORS and not self.M2M_SEARCH_VECTORS:
+ logger.warning("No search_vectors defined for {}".format(
+ self.__class__))
+ return
+ if getattr(self, '_search_updated', None):
+ return
+ self._search_updated = True
+
+ old_search = ""
+ if self.search_vector:
+ old_search = self.search_vector[:]
+ search_vectors = []
+ base_q = self.__class__.objects.filter(pk=self.pk)
+
+ # many to many have to be queried one by one otherwise only one is fetch
+ for M2M_SEARCH_VECTOR in self.M2M_SEARCH_VECTORS:
+ key = M2M_SEARCH_VECTOR.split('__')[0]
+ rel_key = getattr(self, key)
+ for item in rel_key.values('pk').all():
+ query_dct = {key + "__pk": item['pk']}
+ q = copy.copy(base_q).filter(**query_dct)
+ q = q.annotate(
+ search=SearchVector(
+ M2M_SEARCH_VECTOR,
+ config=settings.ISHTAR_SEARCH_LANGUAGE)
+ ).values('search')
+ search_vectors.append(q.all()[0]['search'])
+
+ # int/float are not well managed by the SearchVector
+ for INT_SEARCH_VECTOR in self.INT_SEARCH_VECTORS:
+ q = base_q.values(INT_SEARCH_VECTOR)
+ search_vectors.append(
+ "'{}':1".format(q.all()[0][INT_SEARCH_VECTOR]))
+
+ # copy parent vector fields
+ for PARENT_SEARCH_VECTOR in self.PARENT_SEARCH_VECTORS:
+ parent = getattr(self, PARENT_SEARCH_VECTOR)
+ if hasattr(parent, 'all'): # m2m
+ for p in parent.all():
+ search_vectors.append(p.search_vector)
+ else:
+ search_vectors.append(parent.search_vector)
+
+ # query "simple" fields
+ q = base_q.annotate(
+ search=SearchVector(
+ *self.BASE_SEARCH_VECTORS,
+ config=settings.ISHTAR_SEARCH_LANGUAGE
+ )).values('search')
+ search_vectors.append(q.all()[0]['search'])
+ self.search_vector = merge_tsvectors(search_vectors)
+ changed = old_search != self.search_vector
+ if save and changed:
+ self.skip_history_when_saving = True
+ self.save()
+ return changed
+
+
+class BaseHistorizedItem(FullSearch, Imported, JsonData):
"""
- Historized item with external ID management
+ Historized item with external ID management.
+ All historized items are searcheable and have a data json field
"""
IS_BASKET = False
EXTERNAL_ID_KEY = ''
@@ -1187,6 +1355,7 @@ class LightHistorizedItem(BaseHistorizedItem):
super(LightHistorizedItem, self).save(*args, **kwargs)
return True
+
PARSE_FORMULA = re.compile("{([^}]*)}")
FORMULA_FILTERS = {
@@ -1409,6 +1578,7 @@ def get_current_profile(force=False):
def cached_site_changed(sender, **kwargs):
get_current_profile(force=True)
+
post_save.connect(cached_site_changed, sender=IshtarSiteProfile)
post_delete.connect(cached_site_changed, sender=IshtarSiteProfile)
@@ -2490,12 +2660,20 @@ class Town(Imported, models.Model):
center = models.PointField(_(u"Localisation"), srid=settings.SRID,
blank=True, null=True)
if settings.COUNTRY == 'fr':
- numero_insee = models.CharField(u"Numéro INSEE", max_length=6,
- unique=True)
+ numero_insee = models.CharField(u"Numéro INSEE", max_length=6)
departement = models.ForeignKey(
Department, verbose_name=u"Département", null=True, blank=True)
canton = models.ForeignKey(Canton, verbose_name=u"Canton", null=True,
blank=True)
+ year = models.IntegerField(
+ _("Year of creation"), null=True, blank=True,
+ help_text=_(u"Filling this field is relevant to distinguish old towns "
+ u"to new towns."))
+ children = models.ManyToManyField(
+ 'Town', verbose_name=_(u"Town children"), blank=True,
+ related_name='parents')
+ cached_label = models.CharField(_(u"Cached name"), max_length=500,
+ null=True, blank=True, db_index=True)
objects = models.GeoManager()
class Meta:
@@ -2503,11 +2681,24 @@ class Town(Imported, models.Model):
verbose_name_plural = _(u"Towns")
if settings.COUNTRY == 'fr':
ordering = ['numero_insee']
+ unique_together = (('numero_insee', 'year'),)
def __unicode__(self):
+ if self.cached_label:
+ return self.cached_label
+ self.save()
+ return self.cached_label
+
+ def _generate_cached_label(self):
+ cached_label = self.name
if settings.COUNTRY == "fr":
- return u"%s (%s)" % (self.name, self.numero_insee[:2])
- return self.name
+ cached_label = u"%s - %s" % (self.name, self.numero_insee[:2])
+ if self.year:
+ cached_label += " ({})".format(self.year)
+ return cached_label
+
+
+post_save.connect(cached_label_changed, sender=Town)
class OperationType(GeneralType):
diff --git a/ishtar_common/templates/ishtar/blocks/sheet_json.html b/ishtar_common/templates/ishtar/blocks/sheet_json.html
new file mode 100644
index 000000000..31e6acb84
--- /dev/null
+++ b/ishtar_common/templates/ishtar/blocks/sheet_json.html
@@ -0,0 +1,11 @@
+{% load i18n window_field %}
+{% for json_section, json_fields in item.json_sections %}
+{% if json_section %}
+<h3>{{json_section}}</h3>
+{% endif %}
+{% for label, value in json_fields %}
+{% if forloop.first %}<ul class='form-flex'>{% endif %}
+ {% field_li label value %}
+{% if forloop.last %}</ul>{% endif %}
+{% endfor %}
+{% endfor %}
diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py
index 349408465..bbb449fe3 100644
--- a/ishtar_common/tests.py
+++ b/ishtar_common/tests.py
@@ -23,6 +23,8 @@ import os
import shutil
from StringIO import StringIO
+from django.apps import apps
+
from django.conf import settings
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
@@ -38,6 +40,7 @@ from django.test.runner import DiscoverRunner
from ishtar_common import models
from ishtar_common import views
+from ishtar_common.apps import admin_site
from ishtar_common.utils import post_save_point
@@ -347,6 +350,13 @@ class AdminGenTypeTest(TestCase):
models_with_data = gen_models + [models.ImporterModel]
models = models_with_data
module_name = 'ishtar_common'
+ ishtar_apps = [
+ 'ishtar_common', 'archaeological_files', 'archaeological_operations',
+ 'archaeological_context_records', 'archaeological_warehouse',
+ 'archaeological_finds'
+ ]
+ readonly_models = ['archaeological_finds.Property',
+ 'archaeological_finds.Treatment']
def setUp(self):
self.password = 'mypassword'
@@ -359,16 +369,34 @@ class AdminGenTypeTest(TestCase):
self.client.login(username=self.username, password=self.password)
def test_listing_and_detail(self):
- for model in self.models:
+ models = []
+ for app in self.ishtar_apps:
+ app_models = apps.get_app_config(app).get_models()
+ for model in app_models:
+ if model in admin_site._registry:
+ models.append((app, model))
+ for app, model in models:
# quick test to verify basic access to listing
- base_url = '/admin/{}/{}/'.format(self.module_name,
- model.__name__.lower())
+ base_url = '/admin/{}/{}/'.format(app, model.__name__.lower())
url = base_url
response = self.client.get(url)
self.assertEqual(
response.status_code, 200,
msg="Can not access admin list for {}.".format(model))
- if model in self.models_with_data:
+ nb = model.objects.count()
+ url = base_url + "add/"
+ response = self.client.get(url)
+ if app + "." + model.__name__ in self.readonly_models:
+ continue
+ self.assertEqual(
+ response.status_code, 200,
+ msg="Can not access admin add page for {}.".format(model))
+ self.assertEqual(
+ nb, model.objects.count(),
+ msg="A ghost object have been created on access to add page "
+ "for {}.".format(model))
+
+ if nb:
url = base_url + "{}/change/".format(model.objects.all()[0].pk)
response = self.client.get(url)
self.assertEqual(
@@ -1046,6 +1074,15 @@ class IshtarBasicTest(TestCase):
self.assertEqual(response.status_code, 200)
self.assertIn('class="sheet"', response.content)
+ def test_town_cache(self):
+ models.Town.objects.create(name="Sin City", numero_insee="99999")
+ town = models.Town.objects.get(numero_insee="99999")
+ self.assertEqual(town.cached_label, "Sin City - 99")
+ town.year = 2050
+ town.save()
+ town = models.Town.objects.get(numero_insee="99999")
+ self.assertEqual(town.cached_label, "Sin City - 99 (2050)")
+
class GeomaticTest(TestCase):
def test_post_save_point(self):
diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py
index c6a4032f0..5d9e85c60 100644
--- a/ishtar_common/utils.py
+++ b/ishtar_common/utils.py
@@ -104,9 +104,12 @@ def cached_label_changed(sender, **kwargs):
setattr(instance, cached_label, lbl)
changed = True
if changed:
+ instance._search_updated = False
if hasattr(instance, '_cascade_change') and instance._cascade_change:
instance.skip_history_when_saving = True
instance.save()
+ if hasattr(instance, 'update_search_vector'):
+ instance.update_search_vector()
updated = False
if hasattr(instance, '_cached_labels_bulk_update'):
updated = instance._cached_labels_bulk_update()
@@ -117,6 +120,7 @@ def cached_label_changed(sender, **kwargs):
item.test_obj = instance.test_obj
cached_label_changed(item.__class__, instance=item)
+
SHORTIFY_STR = ugettext(" (...)")
@@ -289,3 +293,43 @@ def get_all_related_objects(model):
and f.auto_created and not f.concrete
]
+
+def merge_tsvectors(vectors):
+ """
+ Parse tsvector to merge them in one string
+ :param vectors: list of tsvector string
+ :return: merged tsvector
+ """
+ result_dict = {}
+ for vector in vectors:
+ if not vector:
+ continue
+
+ current_position = 0
+ if result_dict:
+ for key in result_dict:
+ max_position = max(result_dict[key])
+ if max_position > current_position:
+ current_position = max_position
+
+ for dct_member in vector.split(" "):
+ splitted = dct_member.split(':')
+ key = ":".join(splitted[:-1])
+ positions = splitted[-1]
+ key = key[1:-1] # remove quotes
+ positions = [int(pos) + current_position
+ for pos in positions.split(',')]
+ if key in result_dict:
+ result_dict[key] += positions
+ else:
+ result_dict[key] = positions
+
+ # {'lamelie': [1, 42, 5]} => {'lamelie': "1,42,5"}
+ result_dict = {k: ",".join([str(val) for val in result_dict[k]])
+ for k in result_dict}
+ # {'lamelie': "1,5", "hagarde": "2", "regarde": "4"} =>
+ # "'lamelie':1,5 'hagarde':2 'regarde':4"
+ result = " ".join(["'{}':{}".format(k, result_dict[k])
+ for k in result_dict])
+
+ return result
diff --git a/ishtar_common/views.py b/ishtar_common/views.py
index 997acd7df..b8350c62a 100644
--- a/ishtar_common/views.py
+++ b/ishtar_common/views.py
@@ -41,6 +41,7 @@ from extra_views import ModelFormSetView
from django.conf import settings
from django.contrib.auth import logout
from django.contrib.auth.decorators import login_required
+from django.contrib.postgres.search import SearchQuery
from django.core.exceptions import ObjectDoesNotExist
from django.core.urlresolvers import reverse, NoReverseMatch
from django.db.models import Q, ImageField
@@ -814,6 +815,23 @@ def get_item(model, func_name, default_name, extra_request_keys=[],
dct.pop(k)
# manage hierarchic conditions
for req in dct.copy():
+ if req.endswith('town__pk') or req.endswith('towns__pk'):
+ val = dct.pop(req)
+ reqs = Q(**{req: val})
+ base_req = req[:-2] + '__'
+ req = base_req[:]
+ for idx in range(HIERARCHIC_LEVELS):
+ req = req[:-2] + 'parents__pk'
+ q = Q(**{req: val})
+ reqs |= q
+ req = base_req[:]
+ for idx in range(HIERARCHIC_LEVELS):
+ req = req[:-2] + 'children__pk'
+ q = Q(**{req: val})
+ reqs |= q
+ and_reqs.append(reqs)
+ continue
+
for k_hr in HIERARCHIC_FIELDS:
if type(req) in (list, tuple):
val = dct.pop(req)
@@ -829,12 +847,15 @@ def get_item(model, func_name, default_name, extra_request_keys=[],
val = dct.pop(req)
reqs = Q(**{req: val})
req = req[:-2] + '__'
- for idx in xrange(HIERARCHIC_LEVELS):
+ for idx in range(HIERARCHIC_LEVELS):
req = req[:-2] + 'parent__pk'
q = Q(**{req: val})
reqs |= q
and_reqs.append(reqs)
break
+ if 'search_vector' in dct:
+ dct['search_vector'] = SearchQuery(
+ dct['search_vector'], config=settings.ISHTAR_SEARCH_LANGUAGE)
query = Q(**dct)
for k, or_req in or_reqs:
alt_dct = dct.copy()
@@ -908,6 +929,9 @@ def get_item(model, func_name, default_name, extra_request_keys=[],
items = model.objects.filter(query).distinct()
# print(items.query)
+ if 'search_vector' in dct: # for serialization
+ dct['search_vector'] = dct['search_vector'].value
+
# table cols
if own_table_cols:
table_cols = own_table_cols
diff --git a/ishtar_common/wizards.py b/ishtar_common/wizards.py
index 701f6eca3..f86e03df0 100644
--- a/ishtar_common/wizards.py
+++ b/ishtar_common/wizards.py
@@ -737,6 +737,9 @@ class Wizard(NamedUrlWizardView):
if has_problemetic_null:
continue
+ if hasattr(model, 'data') and 'data' not in value:
+ value['data'] = {}
+
if get_or_create:
value, created = model.objects.get_or_create(
**value)
generated by cgit v1.2.3 (git 2.39.1) at 2025-10-16 23:12:40 +0000