diff options
Diffstat (limited to 'ishtar_common')
20 files changed, 841 insertions, 49 deletions
diff --git a/ishtar_common/admin.py b/ishtar_common/admin.py index cec61a51e..2df910ffd 100644 --- a/ishtar_common/admin.py +++ b/ishtar_common/admin.py @@ -20,11 +20,14 @@ import csv from ajax_select import make_ajax_form +from ajax_select.fields import AutoCompleteSelectField, \ + AutoCompleteSelectMultipleField from django.conf import settings from django.contrib import admin from django.contrib.auth.admin import GroupAdmin, UserAdmin from django.contrib.auth.models import Group, User +from django.contrib.contenttypes.models import ContentType from django.contrib.sites.admin import SiteAdmin from django.contrib.sites.models import Site from django.contrib.gis.forms import PointField, OSMWidget @@ -114,12 +117,22 @@ def export_as_csv_action(description=_(u"Export selected as CSV file"), class HistorizedObjectAdmin(admin.ModelAdmin): - readonly_fields = ['history_creator', 'history_modifier',] + readonly_fields = ['history_creator', 'history_modifier', 'search_vector'] def save_model(self, request, obj, form, change): obj.history_modifier = request.user obj.save() + def get_readonly_fields(self, request, obj=None): + if obj: # editing an existing object + return tuple(self.readonly_fields or []) + tuple(['imports']) + return self.readonly_fields + + def get_exclude(self, request, obj=None): + if not obj: + return tuple(self.exclude or []) + tuple(['imports']) + return self.exclude + class MyGroupAdmin(GroupAdmin): class Media: @@ -153,7 +166,6 @@ class OrganizationAdmin(HistorizedObjectAdmin): list_filter = ("organization_type",) search_fields = ('name',) exclude = ('merge_key', 'merge_exclusion', 'merge_candidate', ) - readonly_fields = HistorizedObjectAdmin.readonly_fields + ['imports'] model = models.Organization admin_site.register(models.Organization, OrganizationAdmin) @@ -164,31 +176,51 @@ class PersonAdmin(HistorizedObjectAdmin): list_filter = ("person_types",) search_fields = ('name', 'surname', 'email', 'raw_name') exclude = ('merge_key', 'merge_exclusion', 'merge_candidate', ) - readonly_fields = HistorizedObjectAdmin.readonly_fields + ['imports'] form = make_ajax_form(models.Person, {'attached_to': 'organization'}) model = models.Person admin_site.register(models.Person, PersonAdmin) +class AdminRelatedTownForm(forms.ModelForm): + class Meta: + model = models.Town.children.through + exclude = [] + from_town = AutoCompleteSelectField( + 'town', required=True, label=_(u"Parent")) + + class AdminTownForm(forms.ModelForm): class Meta: model = models.Town - exclude = [] + exclude = ['imports'] center = PointField(label=_(u"center"), required=False, widget=OSMWidget) + children = AutoCompleteSelectMultipleField('town', required=False, + label=_(u"Town children")) + + +class TownParentInline(admin.TabularInline): + model = models.Town.children.through + fk_name = 'to_town' + form = AdminRelatedTownForm + verbose_name = _(u"Parent") + verbose_name_plural = _(u"Parents") + extra = 1 class TownAdmin(admin.ModelAdmin): + model = models.Town list_display = ['name', ] search_fields = ['name'] + readonly_fields = ['cached_label'] if settings.COUNTRY == 'fr': list_display += ['numero_insee', 'departement', ] search_fields += ['numero_insee', 'departement__label', ] list_filter = ("departement",) - readonly_fields = ['imports'] - model = models.Town form = AdminTownForm + inlines = [TownParentInline] + admin_site.register(models.Town, TownAdmin) @@ -333,6 +365,56 @@ class ItemKeyAdmin(admin.ModelAdmin): admin_site.register(models.ItemKey, ItemKeyAdmin) +class JsonContentTypeFormMixin(object): + class Meta: + model = models.JsonDataSection + exclude = [] + + def __init__(self, *args, **kwargs): + super(JsonContentTypeFormMixin, self).__init__(*args, **kwargs) + choices = [] + for pk, label in self.fields['content_type'].choices: + if not pk: + choices.append((pk, label)) + continue + ct = ContentType.objects.get(pk=pk) + model_class = ct.model_class() + if hasattr(model_class, 'data') and \ + not hasattr(model_class, 'history_type'): + choices.append((pk, label)) + self.fields['content_type'].choices = sorted(choices, + key=lambda x: x[1]) + + +class JsonDataSectionForm(JsonContentTypeFormMixin, forms.ModelForm): + class Meta: + model = models.JsonDataSection + exclude = [] + + +class JsonDataSectionAdmin(admin.ModelAdmin): + list_display = ['name', 'content_type', 'order'] + form = JsonDataSectionForm + + +admin_site.register(models.JsonDataSection, JsonDataSectionAdmin) + + +class JsonDataFieldForm(JsonContentTypeFormMixin, forms.ModelForm): + class Meta: + model = models.JsonDataField + exclude = [] + + +class JsonDataFieldAdmin(admin.ModelAdmin): + list_display = ['name', 'content_type', 'key', 'display', + 'order', 'section'] + form = JsonDataFieldForm + + +admin_site.register(models.JsonDataField, JsonDataFieldAdmin) + + class AdministrationScriptAdmin(admin.ModelAdmin): list_display = ['name', 'path'] diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index 9caebb2dd..e8ec43ab2 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -1486,6 +1486,9 @@ class Importer(object): # importer trigger self._set_importer_trigger(cls, attribute, data) return + if attribute == 'data': # json field + # no need to do anything + return try: field_object = cls._meta.get_field(attribute) except FieldDoesNotExist: @@ -1570,8 +1573,8 @@ class Importer(object): create_dict = copy.deepcopy(data) for k in create_dict.keys(): - # filter unnecessary default values - if type(create_dict[k]) == dict: + # filter unnecessary default values but not the json field + if type(create_dict[k]) == dict and k != 'data': create_dict.pop(k) # File doesn't like deepcopy elif type(create_dict[k]) == File: diff --git a/ishtar_common/management/commands/update_search_vectors.py b/ishtar_common/management/commands/update_search_vectors.py new file mode 100644 index 000000000..c73a6e88e --- /dev/null +++ b/ishtar_common/management/commands/update_search_vectors.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys + +from django.core.management.base import BaseCommand +import django.apps + + +class Command(BaseCommand): + help = "./manage.py update_search_vectors\n\n"\ + "Update full texte search vectors." + + def handle(self, *args, **options): + for model in django.apps.apps.get_models(): + if hasattr(model, "update_search_vector") and \ + getattr(model, "BASE_SEARCH_VECTORS", None): + self.stdout.write("\n* update {}".format(model)) + total = model.objects.count() + for idx, item in enumerate(model.objects.all()): + sys.stdout.write("\r{}/{} ".format(idx, total)) + sys.stdout.flush() + item.update_search_vector() + self.stdout.write("\n") diff --git a/ishtar_common/migrations/0015_auto_20171011_1644.py b/ishtar_common/migrations/0015_auto_20171011_1644.py new file mode 100644 index 000000000..a9f4499c2 --- /dev/null +++ b/ishtar_common/migrations/0015_auto_20171011_1644.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2017-10-11 16:44 +from __future__ import unicode_literals + +import django.contrib.postgres.search +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0014_ishtarsiteprofile_preservation'), + ] + + operations = [ + migrations.AddField( + model_name='historicalorganization', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='historicalperson', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='organization', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + migrations.AddField( + model_name='person', + name='search_vector', + field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'), + ), + ] diff --git a/ishtar_common/migrations/0016_auto_20171016_1104.py b/ishtar_common/migrations/0016_auto_20171016_1104.py new file mode 100644 index 000000000..1d9209bdd --- /dev/null +++ b/ishtar_common/migrations/0016_auto_20171016_1104.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2017-10-16 11:04 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0015_auto_20171011_1644'), + ] + + operations = [ + migrations.AddField( + model_name='town', + name='cached_label', + field=models.CharField(blank=True, db_index=True, max_length=500, null=True, verbose_name='Cached name'), + ), + migrations.AddField( + model_name='town', + name='children', + field=models.ManyToManyField(blank=True, related_name='parents', to='ishtar_common.Town', verbose_name='Town children'), + ), + migrations.AddField( + model_name='town', + name='year', + field=models.IntegerField(blank=True, help_text='If not filled considered as the older town known.', null=True, verbose_name='Year of creation'), + ), + ] diff --git a/ishtar_common/migrations/0017_auto_20171016_1320.py b/ishtar_common/migrations/0017_auto_20171016_1320.py new file mode 100644 index 000000000..a48b36ce7 --- /dev/null +++ b/ishtar_common/migrations/0017_auto_20171016_1320.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2017-10-16 13:20 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0016_auto_20171016_1104'), + ] + + operations = [ + migrations.AlterField( + model_name='town', + name='numero_insee', + field=models.CharField(max_length=6, verbose_name='Num\xe9ro INSEE'), + ), + migrations.AlterField( + model_name='town', + name='year', + field=models.IntegerField(blank=True, help_text='Filling this field is relevant to distinguish old towns to new towns.', null=True, verbose_name='Year of creation'), + ), + migrations.AlterUniqueTogether( + name='town', + unique_together=set([('numero_insee', 'year')]), + ), + ] diff --git a/ishtar_common/migrations/0018_auto_20171017_1840.py b/ishtar_common/migrations/0018_auto_20171017_1840.py new file mode 100644 index 000000000..0c617a3d5 --- /dev/null +++ b/ishtar_common/migrations/0018_auto_20171017_1840.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2017-10-17 18:40 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('ishtar_common', '0017_auto_20171016_1320'), + ] + + operations = [ + migrations.CreateModel( + name='JsonDataField', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=200, verbose_name='Name')), + ('key', models.CharField(help_text='Value of the key in the JSON schema. For hierarchical key use "__" to explain it. For instance the key \'my_subkey\' with data such as {\'my_key\': {\'my_subkey\': \'value\'}} will be reached with my_key__my_subkey.', max_length=200, verbose_name='Key')), + ('display', models.BooleanField(default=True, verbose_name='Display')), + ('order', models.IntegerField(default=10, verbose_name='Order')), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'verbose_name': 'Json data - Field', + 'verbose_name_plural': 'Json data - Fields', + }, + ), + migrations.CreateModel( + name='JsonDataSection', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=200, verbose_name='Name')), + ('order', models.IntegerField(default=10, verbose_name='Order')), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + options={ + 'ordering': ['name'], + 'verbose_name': 'Json data - Menu', + 'verbose_name_plural': 'Json data - Menus', + }, + ), + migrations.AddField( + model_name='historicalorganization', + name='data', + field=django.contrib.postgres.fields.jsonb.JSONField(db_index=True, default={}), + ), + migrations.AddField( + model_name='historicalperson', + name='data', + field=django.contrib.postgres.fields.jsonb.JSONField(db_index=True, default={}), + ), + migrations.AddField( + model_name='organization', + name='data', + field=django.contrib.postgres.fields.jsonb.JSONField(db_index=True, default={}), + ), + migrations.AddField( + model_name='person', + name='data', + field=django.contrib.postgres.fields.jsonb.JSONField(db_index=True, default={}), + ), + migrations.AddField( + model_name='jsondatafield', + name='section', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.JsonDataSection'), + ), + ] diff --git a/ishtar_common/models.py b/ishtar_common/models.py index 28a24115b..c3ba4fdd0 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -35,6 +35,8 @@ import tempfile import time from django.conf import settings +from django.contrib.postgres.fields import JSONField +from django.contrib.postgres.search import SearchVectorField, SearchVector from django.core.cache import cache from django.core.exceptions import ObjectDoesNotExist, ValidationError from django.core.files.uploadedfile import SimpleUploadedFile @@ -58,7 +60,7 @@ from simple_history.models import HistoricalRecords as BaseHistoricalRecords from ishtar_common.model_merging import merge_model_objects from ishtar_common.utils import get_cache, disable_for_loaddata, create_slug,\ - get_all_field_names + get_all_field_names, merge_tsvectors, cached_label_changed from ishtar_common.models_imports import ImporterModel, ImporterType, \ ImporterDefault, ImporterDefaultValues, ImporterColumn, \ @@ -908,6 +910,96 @@ class BulkUpdatedItem(object): return transaction_id, False +class JsonDataSection(models.Model): + content_type = models.ForeignKey(ContentType) + name = models.CharField(_(u"Name"), max_length=200) + order = models.IntegerField(_(u"Order"), default=10) + + class Meta: + verbose_name = _(u"Json data - Menu") + verbose_name_plural = _(u"Json data - Menus") + ordering = ['order', 'name'] + + def __unicode__(self): + return u"{} - {}".format(self.content_type, self.name) + + +class JsonDataField(models.Model): + name = models.CharField(_(u"Name"), max_length=200) + content_type = models.ForeignKey(ContentType) + key = models.CharField( + _(u"Key"), max_length=200, + help_text=_(u"Value of the key in the JSON schema. For hierarchical " + u"key use \"__\" to explain it. For instance the key " + u"'my_subkey' with data such as {'my_key': {'my_subkey': " + u"'value'}} will be reached with my_key__my_subkey.")) + display = models.BooleanField(_(u"Display"), default=True) + order = models.IntegerField(_(u"Order"), default=10) + section = models.ForeignKey(JsonDataSection, blank=True, null=True) + + class Meta: + verbose_name = _(u"Json data - Field") + verbose_name_plural = _(u"Json data - Fields") + ordering = ['order', 'name'] + + def __unicode__(self): + return u"{} - {}".format(self.content_type, self.name) + + def clean(self): + if not self.section: + return + if self.section.content_type != self.content_type: + raise ValidationError( + _(u"Content type of the field and of the menu do not match")) + + +class JsonData(models.Model): + data = JSONField(default={}, db_index=True, blank=True) + + class Meta: + abstract = True + + def pre_save(self): + if not self.data: + self.data = {} + + @property + def json_sections(self): + sections = [] + try: + content_type = ContentType.objects.get_for_model(self) + except ContentType.DoesNotExists: + return sections + fields = list(JsonDataField.objects.filter( + content_type=content_type, display=True, section__isnull=True + ).all()) # no section fields + + fields += list(JsonDataField.objects.filter( + content_type=content_type, display=True, section__isnull=False + ).order_by('section__order', 'order').all()) + + for field in fields: + value = None + data = self.data.copy() + for key in field.key.split('__'): + if key in data: + value = copy.copy(data[key]) + data = data[key] + else: + value = None + break + if not value: + continue + if type(value) in (list, tuple): + value = u" ; ".join([unicode(v) for v in value]) + section_name = field.section.name if field.section else None + if not sections or section_name != sections[-1][0]: + # if section name is identical it is the same + sections.append((section_name, [])) + sections[-1][1].append((field.name, value)) + return sections + + class Imported(models.Model): imports = models.ManyToManyField( Import, blank=True, @@ -917,9 +1009,85 @@ class Imported(models.Model): abstract = True -class BaseHistorizedItem(Imported): +class FullSearch(models.Model): + search_vector = SearchVectorField(_("Search vector"), blank=True, null=True, + help_text=_("Auto filled at save")) + BASE_SEARCH_VECTORS = [] + INT_SEARCH_VECTORS = [] + M2M_SEARCH_VECTORS = [] + PARENT_SEARCH_VECTORS = [] + + class Meta: + abstract = True + + def update_search_vector(self, save=True): + """ + Update the search vector + :param save: True if you want to save the object immediately + :return: True if modified + """ + if not self.BASE_SEARCH_VECTORS and not self.M2M_SEARCH_VECTORS: + logger.warning("No search_vectors defined for {}".format( + self.__class__)) + return + if getattr(self, '_search_updated', None): + return + self._search_updated = True + + old_search = "" + if self.search_vector: + old_search = self.search_vector[:] + search_vectors = [] + base_q = self.__class__.objects.filter(pk=self.pk) + + # many to many have to be queried one by one otherwise only one is fetch + for M2M_SEARCH_VECTOR in self.M2M_SEARCH_VECTORS: + key = M2M_SEARCH_VECTOR.split('__')[0] + rel_key = getattr(self, key) + for item in rel_key.values('pk').all(): + query_dct = {key + "__pk": item['pk']} + q = copy.copy(base_q).filter(**query_dct) + q = q.annotate( + search=SearchVector( + M2M_SEARCH_VECTOR, + config=settings.ISHTAR_SEARCH_LANGUAGE) + ).values('search') + search_vectors.append(q.all()[0]['search']) + + # int/float are not well managed by the SearchVector + for INT_SEARCH_VECTOR in self.INT_SEARCH_VECTORS: + q = base_q.values(INT_SEARCH_VECTOR) + search_vectors.append( + "'{}':1".format(q.all()[0][INT_SEARCH_VECTOR])) + + # copy parent vector fields + for PARENT_SEARCH_VECTOR in self.PARENT_SEARCH_VECTORS: + parent = getattr(self, PARENT_SEARCH_VECTOR) + if hasattr(parent, 'all'): # m2m + for p in parent.all(): + search_vectors.append(p.search_vector) + else: + search_vectors.append(parent.search_vector) + + # query "simple" fields + q = base_q.annotate( + search=SearchVector( + *self.BASE_SEARCH_VECTORS, + config=settings.ISHTAR_SEARCH_LANGUAGE + )).values('search') + search_vectors.append(q.all()[0]['search']) + self.search_vector = merge_tsvectors(search_vectors) + changed = old_search != self.search_vector + if save and changed: + self.skip_history_when_saving = True + self.save() + return changed + + +class BaseHistorizedItem(FullSearch, Imported, JsonData): """ - Historized item with external ID management + Historized item with external ID management. + All historized items are searcheable and have a data json field """ IS_BASKET = False EXTERNAL_ID_KEY = '' @@ -1187,6 +1355,7 @@ class LightHistorizedItem(BaseHistorizedItem): super(LightHistorizedItem, self).save(*args, **kwargs) return True + PARSE_FORMULA = re.compile("{([^}]*)}") FORMULA_FILTERS = { @@ -1409,6 +1578,7 @@ def get_current_profile(force=False): def cached_site_changed(sender, **kwargs): get_current_profile(force=True) + post_save.connect(cached_site_changed, sender=IshtarSiteProfile) post_delete.connect(cached_site_changed, sender=IshtarSiteProfile) @@ -2490,12 +2660,20 @@ class Town(Imported, models.Model): center = models.PointField(_(u"Localisation"), srid=settings.SRID, blank=True, null=True) if settings.COUNTRY == 'fr': - numero_insee = models.CharField(u"Numéro INSEE", max_length=6, - unique=True) + numero_insee = models.CharField(u"Numéro INSEE", max_length=6) departement = models.ForeignKey( Department, verbose_name=u"Département", null=True, blank=True) canton = models.ForeignKey(Canton, verbose_name=u"Canton", null=True, blank=True) + year = models.IntegerField( + _("Year of creation"), null=True, blank=True, + help_text=_(u"Filling this field is relevant to distinguish old towns " + u"to new towns.")) + children = models.ManyToManyField( + 'Town', verbose_name=_(u"Town children"), blank=True, + related_name='parents') + cached_label = models.CharField(_(u"Cached name"), max_length=500, + null=True, blank=True, db_index=True) objects = models.GeoManager() class Meta: @@ -2503,11 +2681,24 @@ class Town(Imported, models.Model): verbose_name_plural = _(u"Towns") if settings.COUNTRY == 'fr': ordering = ['numero_insee'] + unique_together = (('numero_insee', 'year'),) def __unicode__(self): + if self.cached_label: + return self.cached_label + self.save() + return self.cached_label + + def _generate_cached_label(self): + cached_label = self.name if settings.COUNTRY == "fr": - return u"%s (%s)" % (self.name, self.numero_insee[:2]) - return self.name + cached_label = u"%s - %s" % (self.name, self.numero_insee[:2]) + if self.year: + cached_label += " ({})".format(self.year) + return cached_label + + +post_save.connect(cached_label_changed, sender=Town) class OperationType(GeneralType): diff --git a/ishtar_common/static/gentium/GentiumPlus-I.ttf b/ishtar_common/static/gentium/GentiumPlus-I.ttf Binary files differnew file mode 100644 index 000000000..7bc1b3d8b --- /dev/null +++ b/ishtar_common/static/gentium/GentiumPlus-I.ttf diff --git a/ishtar_common/static/gentium/GentiumPlus-R.ttf b/ishtar_common/static/gentium/GentiumPlus-R.ttf Binary files differnew file mode 100644 index 000000000..c1194dd35 --- /dev/null +++ b/ishtar_common/static/gentium/GentiumPlus-R.ttf diff --git a/ishtar_common/static/gentium/OFL.txt b/ishtar_common/static/gentium/OFL.txt new file mode 100644 index 000000000..4f7540787 --- /dev/null +++ b/ishtar_common/static/gentium/OFL.txt @@ -0,0 +1,94 @@ +Copyright (c) 2003-2014 SIL International (http://www.sil.org/),
+with Reserved Font Names "Gentium" and "SIL".
+
+This Font Software is licensed under the SIL Open Font License, Version 1.1.
+This license is copied below, and is also available with a FAQ at:
+http://scripts.sil.org/OFL
+
+
+-----------------------------------------------------------
+SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
+-----------------------------------------------------------
+
+PREAMBLE
+The goals of the Open Font License (OFL) are to stimulate worldwide
+development of collaborative font projects, to support the font creation
+efforts of academic and linguistic communities, and to provide a free and
+open framework in which fonts may be shared and improved in partnership
+with others.
+
+The OFL allows the licensed fonts to be used, studied, modified and
+redistributed freely as long as they are not sold by themselves. The
+fonts, including any derivative works, can be bundled, embedded,
+redistributed and/or sold with any software provided that any reserved
+names are not used by derivative works. The fonts and derivatives,
+however, cannot be released under any other type of license. The
+requirement for fonts to remain under this license does not apply
+to any document created using the fonts or their derivatives.
+
+DEFINITIONS
+"Font Software" refers to the set of files released by the Copyright
+Holder(s) under this license and clearly marked as such. This may
+include source files, build scripts and documentation.
+
+"Reserved Font Name" refers to any names specified as such after the
+copyright statement(s).
+
+"Original Version" refers to the collection of Font Software components as
+distributed by the Copyright Holder(s).
+
+"Modified Version" refers to any derivative made by adding to, deleting,
+or substituting -- in part or in whole -- any of the components of the
+Original Version, by changing formats or by porting the Font Software to a
+new environment.
+
+"Author" refers to any designer, engineer, programmer, technical
+writer or other person who contributed to the Font Software.
+
+PERMISSION & CONDITIONS
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of the Font Software, to use, study, copy, merge, embed, modify,
+redistribute, and sell modified and unmodified copies of the Font
+Software, subject to the following conditions:
+
+1) Neither the Font Software nor any of its individual components,
+in Original or Modified Versions, may be sold by itself.
+
+2) Original or Modified Versions of the Font Software may be bundled,
+redistributed and/or sold with any software, provided that each copy
+contains the above copyright notice and this license. These can be
+included either as stand-alone text files, human-readable headers or
+in the appropriate machine-readable metadata fields within text or
+binary files as long as those fields can be easily viewed by the user.
+
+3) No Modified Version of the Font Software may use the Reserved Font
+Name(s) unless explicit written permission is granted by the corresponding
+Copyright Holder. This restriction only applies to the primary font name as
+presented to the users.
+
+4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
+Software shall not be used to promote, endorse or advertise any
+Modified Version, except to acknowledge the contribution(s) of the
+Copyright Holder(s) and the Author(s) or with their explicit written
+permission.
+
+5) The Font Software, modified or unmodified, in part or in whole,
+must be distributed entirely under this license, and must not be
+distributed under any other license. The requirement for fonts to
+remain under this license does not apply to any document created
+using the Font Software.
+
+TERMINATION
+This license becomes null and void if any of the above conditions are
+not met.
+
+DISCLAIMER
+THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
+OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
+COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
+DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
+OTHER DEALINGS IN THE FONT SOFTWARE.
diff --git a/ishtar_common/static/gentium/README.txt b/ishtar_common/static/gentium/README.txt new file mode 100644 index 000000000..bc17a8cb7 --- /dev/null +++ b/ishtar_common/static/gentium/README.txt @@ -0,0 +1,88 @@ +README
+Gentium Plus
+========================
+
+Thank you for your interest in the Gentium Plus fonts.
+We hope you find them useful!
+
+Gentium Plus supports a wide range of Latin, Greek and Cyrillic
+characters. Documentation for the fonts is available on Gentium website
+(http://scripts.sil.org/Gentium), including details on what ranges are
+supported.
+
+Gentium Plus is released under the SIL Open Font License.
+
+See the OFL and OFL-FAQ for details of the SIL Open Font License.
+See the FONTLOG for information on this and previous releases.
+See the GENTIUM-FAQ for answers to common questions about the Gentium fonts
+See the website (http://scripts.sil.org/Gentium) for further documentation.
+See the SIL Unicode Roman FAQ (http://scripts.sil.org/ComplexRomanFontFAQ)
+for frequently asked questions and their answers regarding SIL's Roman fonts.
+
+
+TIPS
+====
+
+As this font is distributed at no cost, we are unable to provide a
+commercial level of personal technical support. The font has, however,
+been through some testing on various platforms to be sure it works in most
+situations. In particular, it has been tested and shown to work on Windows
+XP, Windows Vista and Windows 7. Graphite capabilities have been tested
+on Graphite-supported platforms.
+
+If you do find a problem, please do report it to fonts@sil.org.
+We can't guarantee any direct response, but will try to fix reported bugs in
+future versions. Make sure you read through the
+SIL Unicode Roman FAQ (http://scripts.sil.org/ComplexRomanFontFAQ).
+
+Many problems can be solved, or at least explained, through an understanding
+of the encoding and use of the fonts. Here are some basic hints:
+
+Encoding:
+The fonts are encoded according to Unicode, so your application must support
+Unicode text in order to access letters other than the standard alphabet.
+Most Windows applications provide basic Unicode support. You will, however,
+need some way of entering Unicode text into your document.
+
+Keyboarding:
+This font does not include any keyboarding helps or utilities. It uses the
+built-in keyboards of the operating system. You will need to install the
+appropriate keyboard and input method for the characters of the language you
+wish to use. If you want to enter characters that are not supported by any
+system keyboard, the Keyman program (www.tavultesoft.com) can be helpful
+on Windows systems. Also available for Windows is MSKLC
+(http://www.microsoft.com/globaldev/tools/msklc.mspx).
+For Linux systems such as Ubuntu, KMFL (http://kmfl.sourceforge.net/)
+is available. Ukelele (http://scripts.sil.org/ukelele) is available for
+Mac OS X versions 10.2 and later.
+
+For other platforms, KMFL (http://kmfl.sourceforge.net/),
+XKB (http://www.x.org/wiki/XKB) or Ukelele (http://scripts.sil.org/ukelele)
+can be helpful.
+
+If you want to enter characters that are not supported by any system
+keyboard, and to access the full Unicode range, we suggest you use
+gucharmap, kcharselect on Ubuntu or similar software.
+
+Another method of entering some symbols is provided by a few applications such
+as Adobe InDesign or OpenOffice.org. They can display a glyph palette or input
+dialog that shows all the glyphs (symbols) in a font and allow you to enter
+them by clicking on the glyph you want.
+
+Rendering:
+This font is designed to work with Graphite or Opentype advanced font
+technologies. To take advantage of the advanced typographic
+capabilities of this font, you must be using applications that provide an
+adequate level of support for Graphite or OpenType. See "Applications
+that provide an adequate level of support for SIL Unicode Roman fonts"
+(http://scripts.sil.org/Complex_AdLvSup).
+
+
+CONTACT
+========
+For more information please visit the Gentium page on SIL International's
+Computers and Writing systems website:
+http://scripts.sil.org/Gentium
+
+Support through the website: http://scripts.sil.org/Support
+
diff --git a/ishtar_common/static/media/style_basic.css b/ishtar_common/static/media/style_basic.css index 1d92928dc..d0f5bbe4a 100644 --- a/ishtar_common/static/media/style_basic.css +++ b/ishtar_common/static/media/style_basic.css @@ -1,7 +1,8 @@ @page { size: a4 portrait; - margin: 2.5cm 1cm 2.5cm 1cm; + margin: 2cm 1cm 2.5cm 1cm; background-image: url("images/ishtar-bg.jpg"); + background-repeat: no-repeat; @frame footer { -pdf-frame-content: pdffooter; bottom: 1cm; @@ -16,6 +17,9 @@ margin-right: 1cm; height: 1.5cm; } + @bottom-center { + content: counter(page) "/" counter(pages); + } } label{ @@ -36,6 +40,13 @@ caption, h3{ font-size:1.5em; } +a img { + display: block; + margin-left: auto; + margin-right: auto; + padding:0.5em; +} + th{ text-align:center; border-bottom:2px solid #922; @@ -72,10 +83,21 @@ td{ display:none; } +caption, hr, .tool-left, .tool-right, .display_details, .display_details_inline{ + display: None; + color: transparent; + background-color: transparent; + border-color: transparent; +} + p{ margin:0.2em; } +td{ + background-color: #ddd; +} + #pdffooter, #pdfheader{ text-align:center; } @@ -84,8 +106,15 @@ p{ font-weight:bold; width:100%; border-bottom:1px solid #922; + position: fixed; + top: -0.5cm; } -.display_details, .display_details_inline{ - display: none; +.window-refs{ + text-align:center; + padding:0; + margin:0; + font-size: 0.9em; + width:100%; + display:block; } diff --git a/ishtar_common/templates/ishtar/blocks/sheet_json.html b/ishtar_common/templates/ishtar/blocks/sheet_json.html new file mode 100644 index 000000000..31e6acb84 --- /dev/null +++ b/ishtar_common/templates/ishtar/blocks/sheet_json.html @@ -0,0 +1,11 @@ +{% load i18n window_field %} +{% for json_section, json_fields in item.json_sections %} +{% if json_section %} +<h3>{{json_section}}</h3> +{% endif %} +{% for label, value in json_fields %} +{% if forloop.first %}<ul class='form-flex'>{% endif %} + {% field_li label value %} +{% if forloop.last %}</ul>{% endif %} +{% endfor %} +{% endfor %} diff --git a/ishtar_common/templates/ishtar/sheet_organization_pdf.html b/ishtar_common/templates/ishtar/sheet_organization_pdf.html index 887c7ccb2..2276aa4d1 100644 --- a/ishtar_common/templates/ishtar/sheet_organization_pdf.html +++ b/ishtar_common/templates/ishtar/sheet_organization_pdf.html @@ -1,6 +1,5 @@ {% extends "ishtar/sheet_organization.html" %} {% block header %} -<link rel="stylesheet" href="{{STATIC_URL}}/media/style_basic.css?ver={{VERSION}}" /> {% endblock %} {% block main_head %} {{ block.super }} @@ -10,9 +9,6 @@ Ishtar – {{APP_NAME}} – {{item}} {% endblock %} {%block head_sheet%}{%endblock%} {%block main_foot%} -<div id="pdffooter"> -– <pdf:pagenumber/> – -</div> </body> </html> {%endblock%} diff --git a/ishtar_common/templates/ishtar/sheet_person_pdf.html b/ishtar_common/templates/ishtar/sheet_person_pdf.html index 199892d2f..9dd9e4c50 100644 --- a/ishtar_common/templates/ishtar/sheet_person_pdf.html +++ b/ishtar_common/templates/ishtar/sheet_person_pdf.html @@ -1,6 +1,5 @@ {% extends "ishtar/sheet_person.html" %} {% block header %} -<link rel="stylesheet" href="{{STATIC_URL}}/media/style_basic.css?ver={{VERSION}}" /> {% endblock %} {% block main_head %} {{ block.super }} @@ -10,9 +9,6 @@ Ishtar – {{APP_NAME}} – {{item}} {% endblock %} {%block head_sheet%}{%endblock%} {%block main_foot%} -<div id="pdffooter"> -– <pdf:pagenumber/> – -</div> </body> </html> {%endblock%} diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py index 349408465..bbb449fe3 100644 --- a/ishtar_common/tests.py +++ b/ishtar_common/tests.py @@ -23,6 +23,8 @@ import os import shutil from StringIO import StringIO +from django.apps import apps + from django.conf import settings from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType @@ -38,6 +40,7 @@ from django.test.runner import DiscoverRunner from ishtar_common import models from ishtar_common import views +from ishtar_common.apps import admin_site from ishtar_common.utils import post_save_point @@ -347,6 +350,13 @@ class AdminGenTypeTest(TestCase): models_with_data = gen_models + [models.ImporterModel] models = models_with_data module_name = 'ishtar_common' + ishtar_apps = [ + 'ishtar_common', 'archaeological_files', 'archaeological_operations', + 'archaeological_context_records', 'archaeological_warehouse', + 'archaeological_finds' + ] + readonly_models = ['archaeological_finds.Property', + 'archaeological_finds.Treatment'] def setUp(self): self.password = 'mypassword' @@ -359,16 +369,34 @@ class AdminGenTypeTest(TestCase): self.client.login(username=self.username, password=self.password) def test_listing_and_detail(self): - for model in self.models: + models = [] + for app in self.ishtar_apps: + app_models = apps.get_app_config(app).get_models() + for model in app_models: + if model in admin_site._registry: + models.append((app, model)) + for app, model in models: # quick test to verify basic access to listing - base_url = '/admin/{}/{}/'.format(self.module_name, - model.__name__.lower()) + base_url = '/admin/{}/{}/'.format(app, model.__name__.lower()) url = base_url response = self.client.get(url) self.assertEqual( response.status_code, 200, msg="Can not access admin list for {}.".format(model)) - if model in self.models_with_data: + nb = model.objects.count() + url = base_url + "add/" + response = self.client.get(url) + if app + "." + model.__name__ in self.readonly_models: + continue + self.assertEqual( + response.status_code, 200, + msg="Can not access admin add page for {}.".format(model)) + self.assertEqual( + nb, model.objects.count(), + msg="A ghost object have been created on access to add page " + "for {}.".format(model)) + + if nb: url = base_url + "{}/change/".format(model.objects.all()[0].pk) response = self.client.get(url) self.assertEqual( @@ -1046,6 +1074,15 @@ class IshtarBasicTest(TestCase): self.assertEqual(response.status_code, 200) self.assertIn('class="sheet"', response.content) + def test_town_cache(self): + models.Town.objects.create(name="Sin City", numero_insee="99999") + town = models.Town.objects.get(numero_insee="99999") + self.assertEqual(town.cached_label, "Sin City - 99") + town.year = 2050 + town.save() + town = models.Town.objects.get(numero_insee="99999") + self.assertEqual(town.cached_label, "Sin City - 99 (2050)") + class GeomaticTest(TestCase): def test_post_save_point(self): diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py index c6a4032f0..5d9e85c60 100644 --- a/ishtar_common/utils.py +++ b/ishtar_common/utils.py @@ -104,9 +104,12 @@ def cached_label_changed(sender, **kwargs): setattr(instance, cached_label, lbl) changed = True if changed: + instance._search_updated = False if hasattr(instance, '_cascade_change') and instance._cascade_change: instance.skip_history_when_saving = True instance.save() + if hasattr(instance, 'update_search_vector'): + instance.update_search_vector() updated = False if hasattr(instance, '_cached_labels_bulk_update'): updated = instance._cached_labels_bulk_update() @@ -117,6 +120,7 @@ def cached_label_changed(sender, **kwargs): item.test_obj = instance.test_obj cached_label_changed(item.__class__, instance=item) + SHORTIFY_STR = ugettext(" (...)") @@ -289,3 +293,43 @@ def get_all_related_objects(model): and f.auto_created and not f.concrete ] + +def merge_tsvectors(vectors): + """ + Parse tsvector to merge them in one string + :param vectors: list of tsvector string + :return: merged tsvector + """ + result_dict = {} + for vector in vectors: + if not vector: + continue + + current_position = 0 + if result_dict: + for key in result_dict: + max_position = max(result_dict[key]) + if max_position > current_position: + current_position = max_position + + for dct_member in vector.split(" "): + splitted = dct_member.split(':') + key = ":".join(splitted[:-1]) + positions = splitted[-1] + key = key[1:-1] # remove quotes + positions = [int(pos) + current_position + for pos in positions.split(',')] + if key in result_dict: + result_dict[key] += positions + else: + result_dict[key] = positions + + # {'lamelie': [1, 42, 5]} => {'lamelie': "1,42,5"} + result_dict = {k: ",".join([str(val) for val in result_dict[k]]) + for k in result_dict} + # {'lamelie': "1,5", "hagarde": "2", "regarde": "4"} => + # "'lamelie':1,5 'hagarde':2 'regarde':4" + result = " ".join(["'{}':{}".format(k, result_dict[k]) + for k in result_dict]) + + return result diff --git a/ishtar_common/views.py b/ishtar_common/views.py index 997acd7df..8d475aff5 100644 --- a/ishtar_common/views.py +++ b/ishtar_common/views.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (C) 2010-2016 Étienne Loks <etienne.loks_AT_peacefrogsDOTnet> +# Copyright (C) 2010-2017 Étienne Loks <etienne.loks_AT_peacefrogsDOTnet> # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -21,13 +21,7 @@ from tidylib import tidy_document as tidy from copy import copy, deepcopy import csv -import cStringIO as StringIO import datetime - -import reportlab -reportlab.Version = "2.2" # stupid hack for an old library... -import ho.pisa as pisa - import json import logging from markdown import markdown @@ -35,12 +29,16 @@ import optparse import re from tempfile import NamedTemporaryFile import unicodedata +from weasyprint import HTML, CSS +from weasyprint.fonts import FontConfiguration from extra_views import ModelFormSetView from django.conf import settings from django.contrib.auth import logout from django.contrib.auth.decorators import login_required +from django.contrib.postgres.search import SearchQuery +from django.contrib.staticfiles.templatetags.staticfiles import static from django.core.exceptions import ObjectDoesNotExist from django.core.urlresolvers import reverse, NoReverseMatch from django.db.models import Q, ImageField @@ -814,6 +812,23 @@ def get_item(model, func_name, default_name, extra_request_keys=[], dct.pop(k) # manage hierarchic conditions for req in dct.copy(): + if req.endswith('town__pk') or req.endswith('towns__pk'): + val = dct.pop(req) + reqs = Q(**{req: val}) + base_req = req[:-2] + '__' + req = base_req[:] + for idx in range(HIERARCHIC_LEVELS): + req = req[:-2] + 'parents__pk' + q = Q(**{req: val}) + reqs |= q + req = base_req[:] + for idx in range(HIERARCHIC_LEVELS): + req = req[:-2] + 'children__pk' + q = Q(**{req: val}) + reqs |= q + and_reqs.append(reqs) + continue + for k_hr in HIERARCHIC_FIELDS: if type(req) in (list, tuple): val = dct.pop(req) @@ -829,12 +844,15 @@ def get_item(model, func_name, default_name, extra_request_keys=[], val = dct.pop(req) reqs = Q(**{req: val}) req = req[:-2] + '__' - for idx in xrange(HIERARCHIC_LEVELS): + for idx in range(HIERARCHIC_LEVELS): req = req[:-2] + 'parent__pk' q = Q(**{req: val}) reqs |= q and_reqs.append(reqs) break + if 'search_vector' in dct: + dct['search_vector'] = SearchQuery( + dct['search_vector'], config=settings.ISHTAR_SEARCH_LANGUAGE) query = Q(**dct) for k, or_req in or_reqs: alt_dct = dct.copy() @@ -908,6 +926,9 @@ def get_item(model, func_name, default_name, extra_request_keys=[], items = model.objects.filter(query).distinct() # print(items.query) + if 'search_vector' in dct: # for serialization + dct['search_vector'] = dct['search_vector'].value + # table cols if own_table_cols: table_cols = own_table_cols @@ -1309,19 +1330,25 @@ def show_item(model, name, extra_dct=None): elif doc_type == 'pdf': tpl = loader.get_template('ishtar/sheet_%s_pdf.html' % name) context_instance['output'] = 'PDF' - content = tpl.render(context_instance, request) - result = StringIO.StringIO() - html = content.encode('utf-8') - html = html.replace("<table", "<pdf:nextpage/><table repeat='1'") - pdf = pisa.pisaDocument(StringIO.StringIO(html), result, - encoding='utf-8') - response = HttpResponse(result.getvalue(), - content_type='application/pdf') + html = tpl.render(context_instance, request) + font_config = FontConfiguration() + css = CSS(string=''' + @font-face { + font-family: Gentium; + src: url(%s); + } + body{ + font-family: Gentium + } + ''' % (static("gentium/GentiumPlus-R.ttf"))) + css2 = CSS(filename=settings.STATIC_ROOT + '/media/style_basic.css') + pdf = HTML(string=html, base_url=request.build_absolute_uri() + ).write_pdf(stylesheets=[css, css2], + font_config=font_config) + response = HttpResponse(pdf, content_type='application/pdf') response['Content-Disposition'] = 'attachment; filename=%s.pdf' % \ filename - if not pdf.err: - return response - return HttpResponse(content, content_type="application/xhtml") + return response else: tpl = loader.get_template('ishtar/sheet_%s_window.html' % name) content = tpl.render(context_instance, request) diff --git a/ishtar_common/wizards.py b/ishtar_common/wizards.py index 701f6eca3..f86e03df0 100644 --- a/ishtar_common/wizards.py +++ b/ishtar_common/wizards.py @@ -737,6 +737,9 @@ class Wizard(NamedUrlWizardView): if has_problemetic_null: continue + if hasattr(model, 'data') and 'data' not in value: + value['data'] = {} + if get_or_create: value, created = model.objects.get_or_create( **value) |