Label generation: optimize template evaluation

7 files changed, 215 insertions, 112 deletions

diff --git a/archaeological_context_records/models.py b/archaeological_context_records/models.py
index 8e69ac505..8d69fba4b 100644
--- a/archaeological_context_records/models.py
+++ b/archaeological_context_records/models.py
@@ -133,16 +133,23 @@ class Dating(models.Model):
     def natural_key(self):
         return self.uuid,
 
-    def get_values(self, prefix='', no_values=False):
-        return {
-            prefix + "period": str(self.period),
-            prefix + "start_date": self.start_date or '',
-            prefix + "end_date": self.end_date or "",
-            prefix + "dating_type":
-                str(self.dating_type) if self.dating_type else "",
-            prefix + "quality": str(self.quality) if self.quality else "",
-            prefix + "precise_dating": self.precise_dating
-        }
+    def get_values(self, prefix='', no_values=False, filtr=None, **kwargs):
+        values = {}
+        if not filtr or prefix + "period" in filtr:
+            values[prefix + "period"] = str(self.period)
+        if not filtr or prefix + "start_date" in filtr:
+            values[prefix + "start_date"] = self.start_date or ''
+        if not filtr or prefix + "end_date" in filtr:
+            values[prefix + "end_date"] = self.end_date or ''
+        if not filtr or prefix + "dating_type" in filtr:
+            values[prefix + "dating_type"] = \
+                str(self.dating_type) if self.dating_type else ""
+        if not filtr or prefix + "quality" in filtr:
+            values[prefix + "quality"] = \
+                str(self.quality) if self.quality else ""
+        if not filtr or prefix + "precise_dating" in filtr:
+            values[prefix + "precise_dating"] = self.precise_dating
+        return values
 
     HISTORY_ATTR = ["period", "start_date", "end_date", "dating_type",
                     "quality", "precise_dating"]
@@ -652,15 +659,20 @@ class ContextRecord(BulkUpdatedItem, DocumentItem, BaseHistorizedItem,
         })
         return dct
 
-    def get_values(self, prefix='', no_values=False, no_base_finds=True):
-        values = super(ContextRecord, self).get_values(prefix=prefix,
-                                                       no_values=no_values)
+    def get_values(self, prefix='', no_values=False, filtr=None, **kwargs):
+        no_base_finds = True
+        if "no_base_finds" in kwargs:
+            no_base_finds = kwargs["no_base_finds"]
+        values = super(ContextRecord, self).get_values(
+            prefix=prefix, no_values=no_values, filtr=filtr, **kwargs)
         if prefix and no_base_finds:
             return values
-        values[prefix + 'base_finds'] = [
-            bf.get_values(prefix=prefix, no_values=True)
-            for bf in self.base_finds.distinct().all()
-        ]
+
+        if not filtr or prefix + 'base_finds' in filtr:
+            values[prefix + 'base_finds'] = [
+                bf.get_values(
+                    prefix=prefix, no_values=True, filtr=None, **kwargs)
+                for bf in self.base_finds.distinct().all()]
         return values
 
     def get_town_centroid(self):
diff --git a/archaeological_files/models.py b/archaeological_files/models.py
index cb327f970..328e2c658 100644
--- a/archaeological_files/models.py
+++ b/archaeological_files/models.py
@@ -549,10 +549,10 @@ class File(ClosedItem, DocumentItem, BaseHistorizedItem, OwnPerms, ValueGetter,
             get_short_menu_class=get_short_menu_class)
         return cls._return_get_owns(owns, values, get_short_menu_class)
 
-    def get_values(self, prefix='', no_values=False):
-        values = super(File, self).get_values(prefix=prefix,
-                                              no_values=no_values)
-        return get_values_town_related(self, prefix, values)
+    def get_values(self, prefix='', no_values=False, filtr=None, **kwargs):
+        values = super(File, self).get_values(
+            prefix=prefix, no_values=no_values, filtr=filtr, **kwargs)
+        return get_values_town_related(self, prefix, values, filtr=filtr)
 
     def render_parcels(self):
         from archaeological_operations.models import Parcel
diff --git a/archaeological_finds/models_finds.py b/archaeological_finds/models_finds.py
index 0a32bba69..0260200e9 100644
--- a/archaeological_finds/models_finds.py
+++ b/archaeological_finds/models_finds.py
@@ -357,15 +357,18 @@ class BaseFind(BulkUpdatedItem, BaseHistorizedItem, GeoItem, OwnPerms,
         })
         return dct
 
-    def get_values(self, prefix='', no_values=False, no_find=False):
-        values = super(BaseFind, self).get_values(prefix=prefix,
-                                                  no_values=no_values)
+    def get_values(self, prefix='', no_values=False, filtr=None, **kwargs):
+        no_find = kwargs.get("no_find", False)
+        values = super(BaseFind, self).get_values(
+            prefix=prefix, no_values=no_values, filtr=filtr, **kwargs)
         if no_find:
             return values
-        values[prefix + "finds"] = [
-            find.get_values(no_values=True, no_base_finds=True)
-            for find in self.find.order_by('pk').all()
-        ]
+        if not filtr or prefix + "finds" in filtr:
+            kwargs["no_base_finds"] = True
+            values[prefix + "finds"] = [
+                find.get_values(no_values=True, filtr=None, **kwargs)
+                for find in self.find.order_by('pk').all()
+            ]
         return values
 
     def get_last_find(self):
@@ -665,13 +668,14 @@ class FindBasket(Basket, MainItem, ValueGetter):
             ("view_own_find", u"Can view own Find"),
         )
 
-    def get_values(self, prefix='', no_values=False):
-        values = super(FindBasket, self).get_values(prefix=prefix,
-                                                    no_values=no_values)
-        values[prefix + "items"] = [
-            item.get_values(no_values=True)
-            for item in self.items.distinct().all()
-        ]
+    def get_values(self, prefix='', no_values=False, filtr=None, **kwargs):
+        values = super(FindBasket, self).get_values(
+            prefix=prefix, no_values=no_values, filtr=filtr, **kwargs)
+        if not filtr or prefix + "items" in filtr:
+            values[prefix + "items"] = [
+                item.get_values(no_values=True, filtr=None, **kwargs)
+                for item in self.items.distinct().all()
+            ]
         return values
 
     def get_extra_actions(self, request):
@@ -1761,13 +1765,17 @@ class Find(BulkUpdatedItem, ValueGetter, DocumentItem, BaseHistorizedItem,
             return
         return self.base_finds.order_by('-pk').all()[0]
 
-    def get_values(self, prefix='', no_values=False, no_base_finds=False):
-        values = super(Find, self).get_values(prefix=prefix,
-                                              no_values=no_values)
-        if no_base_finds:
+    def get_values(self, prefix='', no_values=False, filtr=None, **kwargs):
+        no_base_finds = False
+        if "no_base_finds" in kwargs:
+            no_base_finds = kwargs["no_base_finds"]
+        values = super(Find, self).get_values(
+            prefix=prefix, no_values=no_values, filtr=filtr, **kwargs)
+        if no_base_finds or (filtr and prefix + "base_finds" not in filtr):
             return values
+        kwargs["no_find"] = True
         values[prefix + "base_finds"] = [
-            base_find.get_values(no_values=True, no_find=True)
+            base_find.get_values(no_values=True, filtr=None, **kwargs)
             for base_find in self.base_finds.distinct().order_by('-pk').all()
         ]
         return values
diff --git a/archaeological_finds/models_treatments.py b/archaeological_finds/models_treatments.py
index 071d80af1..a7af1117e 100644
--- a/archaeological_finds/models_treatments.py
+++ b/archaeological_finds/models_treatments.py
@@ -318,8 +318,8 @@ class Treatment(DashboardFormItem, ValueGetter, DocumentItem,
         Upstream finds label
         :return: string
         """
-        return u" ; ".join([f.cached_label for f in self.upstream.all()])
-    upstream_lbl.short_description = _(u"Upstream finds")
+        return " ; ".join([f.cached_label for f in self.upstream.all()])
+    upstream_lbl.short_description = _("Upstream finds")
     upstream_lbl.admin_order_field = 'upstream__cached_label'
 
     def get_extra_actions(self, request):
@@ -328,26 +328,31 @@ class Treatment(DashboardFormItem, ValueGetter, DocumentItem,
         if self.can_do(request, 'add_administrativeact'):
             actions += [
                 (reverse('treatment-add-adminact', args=[self.pk]),
-                 _(u"Add associated administrative act"), "fa fa-plus",
-                 _(u"admin. act"), "", False),
+                 _("Add associated administrative act"), "fa fa-plus",
+                 _("admin. act"), "", False),
             ]
         return actions
 
-    def get_values(self, prefix='', no_values=False):
-        values = super(Treatment, self).get_values(prefix=prefix,
-                                                   no_values=no_values)
-        values[prefix + "upstream_finds"] = u" ; ".join(
-            [str(up) for up in self.upstream.all()])
-        values[prefix + "downstream_finds"] = u" ; ".join(
-            [str(down) for down in self.downstream.all()])
-        values[prefix + "operations"] = u" ; ".join(
-            [str(ope) for ope in self.get_query_operations().all()])
-        if self.upstream.count():
+    def get_values(self, prefix='', no_values=False, filtr=None, **kwargs):
+        values = super(Treatment, self).get_values(
+            prefix=prefix, no_values=no_values, filtr=filtr, **kwargs)
+        if not filtr or prefix + "upstream_finds" in filtr:
+            values[prefix + "upstream_finds"] = " ; ".join(
+                [str(up) for up in self.upstream.all()])
+        if not filtr or prefix + "downstream_finds" in filtr:
+            values[prefix + "downstream_finds"] = " ; ".join(
+                [str(up) for up in self.downstream.all()])
+
+        if not filtr or prefix + "operations" in filtr:
+            values[prefix + "operations"] = " ; ".join(
+                [str(ope) for ope in self.get_query_operations().all()])
+        if 'associatedfind_' not in prefix and self.upstream.count():
             find = self.upstream.all()[0]
-            if 'associatedfind_' not in prefix:
-                values.update(
-                    find.get_values(prefix=prefix + 'associatedfind_',
-                                    no_values=True))
+            new_prefix = prefix + 'associatedfind_'
+            new_filtr = self._get_values_update_sub_filter(filtr, new_prefix)
+            values.update(
+                find.get_values(prefix=new_prefix, no_values=True,
+                                filtr=new_filtr, **kwargs))
         return values
 
     def pre_save(self):
diff --git a/archaeological_operations/models.py b/archaeological_operations/models.py
index 3494e6427..014dbb269 100644
--- a/archaeological_operations/models.py
+++ b/archaeological_operations/models.py
@@ -620,24 +620,37 @@ for attr in ArchaeologicalSite.HISTORICAL_M2M:
                         sender=getattr(ArchaeologicalSite, attr).through)
 
 
-def get_values_town_related(item, prefix, values):
-    values[prefix + 'parcellist'] = item.render_parcels()
-    values[prefix + 'towns'] = ''
-    values[prefix + 'departments'] = ''
-    values[prefix + 'departments_number'] = ''
-    values[prefix + 'towns_count'] = str(item.towns.count())
+def get_values_town_related(item, prefix, values, filtr=None):
+    if not filtr or prefix + 'parcellist' in filtr:
+        values[prefix + 'parcellist'] = item.render_parcels()
+    if not filtr or prefix + 'towns_count' in filtr:
+        values[prefix + 'towns_count'] = str(item.towns.count())
+    get_towns = not filtr or prefix + 'towns' in filtr
+    get_dpt = not filtr or prefix + 'departments' in filtr
+    get_dpt_nb = not filtr or prefix + 'departments_number' in filtr
+    if not get_towns and not get_dpt and not get_dpt_nb:
+        return values
+    if get_towns:
+        values[prefix + 'towns'] = ''
+    if get_dpt:
+        values[prefix + 'departments'] = ''
+    if get_dpt_nb:
+        values[prefix + 'departments_number'] = ''
     if item.towns.count():
-        values[prefix + 'towns'] = ", ".join([
-            town.name for town in item.towns.all().order_by('name')])
-        if settings.COUNTRY == 'fr':
+        if get_towns:
+            values[prefix + 'towns'] = ", ".join([
+               town.name for town in item.towns.all().order_by('name')])
+        if settings.COUNTRY == 'fr' and (get_dpt_nb or get_dpt_nb):
             dpts_num = set(
                 [town.numero_insee[:2] for town in item.towns.all()])
-            values[prefix + 'departments_number'] = ", ".join(
-                list(sorted(dpts_num)))
-            values[prefix + 'departments'] = ", ".join(
-                [Department.objects.get(number=dpt).label
-                 for dpt in sorted(dpts_num) if Department.objects.filter(
-                    number=dpt).count()])
+            if get_dpt_nb:
+                values[prefix + 'departments_number'] = ", ".join(
+                    list(sorted(dpts_num)))
+            if get_dpt:
+                values[prefix + 'departments'] = ", ".join(
+                    [Department.objects.get(number=dpt).label
+                     for dpt in sorted(dpts_num) if Department.objects.filter(
+                        number=dpt).count()])
     return values
 
 
@@ -1204,16 +1217,19 @@ class Operation(ClosedItem, DocumentItem, BaseHistorizedItem, QRCodeItem,
     def __str__(self):
         return self.cached_label or ""
 
-    def get_values(self, prefix='', no_values=False):
-        values = super(Operation, self).get_values(prefix=prefix,
-                                                   no_values=no_values)
-        values = get_values_town_related(self, prefix, values)
+    def get_values(self, prefix='', no_values=False, filtr=None, **kwargs):
+        values = super(Operation, self).get_values(
+            prefix=prefix, no_values=no_values, filtr=filtr, **kwargs)
+        values = get_values_town_related(self, prefix, values, filtr=filtr)
         if prefix:
             return values
-        values[prefix + 'context_records'] = [
-            cr.get_values(prefix=prefix, no_values=True, no_base_finds=False)
-            for cr in self.context_record.all()
-        ]
+        if not filtr or prefix + 'context_records' in filtr:
+            kwargs["no_base_finds"] = False
+            values[prefix + 'context_records'] = [
+                cr.get_values(prefix=prefix, no_values=True, filtr=None,
+                              **kwargs)
+                for cr in self.context_record.all()
+            ]
         return values
 
     def public_representation(self):
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index 94d8744e5..f4d898dc5 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -42,6 +42,7 @@ from ooopy.OOoPy import OOoPy
 from ooopy.Transformer import Transformer as OOTransformer
 import ooopy.Transforms as OOTransforms
 import uuid
+import zipfile
 
 from django import forms
 from django.apps import apps
@@ -93,7 +94,7 @@ from ishtar_common.models_imports import ImporterModel, ImporterType, \
 from ishtar_common.templatetags.link_to_window import simple_link_to_window
 from ishtar_common.utils import get_cache, disable_for_loaddata, create_slug, \
     get_all_field_names, merge_tsvectors, cached_label_changed, post_save_geo, \
-    generate_relation_graph, max_size_help, task, SecretaryRenderer
+    generate_relation_graph, max_size_help, task
 
 __all__ = [
     'ImporterModel', 'ImporterType', 'ImporterDefault', 'ImporterDefaultValues',
@@ -136,25 +137,38 @@ class ValueGetter(object):
     GET_VALUES_EXTRA_TYPES = [
         'preservation_to_considers', 'alterations', 'alteration_causes']
 
-    def _get_values_documents(self, prefix=""):
+    def _get_values_documents(self, prefix="", filtr=None):
         values = {}
         if not hasattr(self, 'documents'):
             return values
-        values[prefix + "documents"] = [
-            doc.get_values(no_values=True)
-            for doc in self.documents.all()
-        ]
+        if not filtr or prefix + "documents" in filtr:
+            values[prefix + "documents"] = [
+                doc.get_values(no_values=True)
+                for doc in self.documents.all()
+            ]
+        if filtr and prefix + "main_image" not in filtr:
+            return values
         if hasattr(self, "main_image") and self.main_image and hasattr(
                 self.main_image, "get_values"):
             values[prefix + "main_image"] = self.main_image.get_values(
                 no_values=True)
         return values
 
-    def get_values(self, prefix='', no_values=False):
+    def _get_values_update_sub_filter(self, filtr, prefix):
+        if not filtr:
+            return
+        new_filtr = []
+        for k in filtr:
+            if k.startswith(prefix):
+                new_filtr.append(k[len(prefix):])
+        return new_filtr
+
+    def get_values(self, prefix='', no_values=False, filtr=None, **kwargs):
         if not prefix:
             prefix = self._prefix
         values = {}
-        if hasattr(self, "qrcode"):
+        if hasattr(self, "qrcode") and (not filtr
+                                        or 'qrcode_path' in filtr):
             values['qrcode_path'] = self.qrcode_path
         for field_name in get_all_field_names(self):
             try:
@@ -163,15 +177,27 @@ class ValueGetter(object):
                 continue
             if field_name in self.GET_VALUES_EXCLUDE_FIELDS:
                 continue
+            if filtr and prefix + field_name not in filtr:
+                if hasattr(value, 'get_values'):
+                    new_prefix = prefix + field_name + '_'
+                    new_filtr = self._get_values_update_sub_filter(filtr,
+                                                                   new_prefix)
+                    values.update(
+                        value.get_values(new_prefix, filtr=new_filtr, **kwargs))
+                continue
             if hasattr(self, "get_values_for_" + field_name):
                 values[prefix + field_name] = getattr(
                     self, "get_values_for_" + field_name)()
             else:
                 if hasattr(value, 'get_values'):
-                    values.update(value.get_values(prefix + field_name + '_'))
+                    new_prefix = prefix + field_name + '_'
+                    new_filtr = self._get_values_update_sub_filter(filtr,
+                                                                   new_prefix)
+                    values.update(
+                        value.get_values(new_prefix, filtr=new_filtr, **kwargs))
                 else:
                     values[prefix + field_name] = value
-        values.update(self._get_values_documents(prefix=prefix))
+        values.update(self._get_values_documents(prefix=prefix, filtr=filtr))
         for extra_field in self.GET_VALUES_EXTRA:
             values[prefix + extra_field] = getattr(self, extra_field) or ''
         for key in values.keys():
@@ -3645,6 +3671,23 @@ class DocumentTemplate(models.Model):
             output.write(result)
         return output_name
 
+    def get_filter(self, template, regexp_list=None):
+        if not regexp_list:
+            return None
+        z = zipfile.ZipFile(template)
+        content = z.open('content.xml')
+        full_content = content.read().decode("utf-8")
+        filtr = []
+        for regexp in regexp_list:
+            iter = re.finditer(
+                regexp,
+                full_content)
+            for s in iter:
+                key = s.groups()[0]
+                if key not in filtr:
+                    filtr.append(key)
+        return filtr
+
     def publish_labels(self, objects):
         if not objects:
             return
@@ -3655,6 +3698,13 @@ class DocumentTemplate(models.Model):
         suffix = "." + self.template.name.split('.')[-1]
         len_objects = len(objects)
         names = []
+
+        filtr = self.get_filter(
+            self.template,
+            # e.g.: {{items.4.key}}
+            [r'{{ *items\.\d\.([A-Za-z0-9_.]*)(\|[A-Za-z0-9_.-]*)* *}}']
+        )
+
         for idx in range(int(len(objects) / self.label_per_page) + 1):
             if idx * self.label_per_page >= len_objects:
                 break
@@ -3664,7 +3714,7 @@ class DocumentTemplate(models.Model):
                 if c_idx >= len_objects:
                     break
                 obj = objects[c_idx]
-                values["items"].append(obj.get_values())
+                values["items"].append(obj.get_values(filtr=filtr))
             engine = IshtarSecretaryRenderer()
             try:
                 result = engine.render(self.template, **values)
@@ -3808,12 +3858,18 @@ class Town(Imported, models.Model):
                   'year': self.year or ""}
         return values
 
-    def get_values(self, prefix='', no_values=False, no_base_finds=True):
-        return {
-            prefix or "label": str(self),
-            prefix + "name": self.name,
-            prefix + "numero_insee": self.numero_insee
-        }
+    def get_values(self, prefix='', no_values=False, filtr=None, **kwargs):
+        values = {}
+        if not filtr or prefix in filtr or "label" in filtr:
+            if prefix:
+                values[prefix] = str(self)
+            else:
+                values['label'] = str(self)
+        if not filtr or prefix + "name" in filtr:
+            values[prefix + "name"] = self.name
+        if not filtr or prefix + "numero_insee" in filtr:
+            values[prefix + "numero_insee"] = self.numero_insee
+        return values
 
     @classmethod
     def history_decompress(cls, full_value, create=False):
@@ -4579,9 +4635,9 @@ class Person(Address, Merge, OwnPerms, ValueGetter, MainItem):
             values.append(attached_to)
         return " ".join(values)
 
-    def get_values(self, prefix='', no_values=False):
-        values = super(Person, self).get_values(prefix=prefix,
-                                                no_values=no_values)
+    def get_values(self, prefix='', no_values=False, filtr=None, **kwargs):
+        values = super(Person, self).get_values(
+            prefix=prefix, no_values=no_values, filtr=filtr, **kwargs)
         if not self.attached_to:
             values.update(
                 Person.get_empty_values(prefix=prefix + 'attached_to_'))
@@ -5051,10 +5107,11 @@ class Basket(FullSearch, OwnPerms, ValueGetter, TemplateItem):
         return Q(user=ishtaruser) | Q(
             shared_write_with=ishtaruser)
 
-    def get_values(self, prefix='', no_values=False):
-        values = super(Basket, self).get_values(prefix=prefix,
-                                                no_values=no_values)
-        values["items"] = [item.get_values() for item in self.items.all()]
+    def get_values(self, prefix='', no_values=False, filtr=None, **kwargs):
+        values = super(Basket, self).get_values(
+            prefix=prefix, no_values=no_values, filtr=filtr, **kwargs)
+        if not filtr or "items" in filtr:
+            values["items"] = [item.get_values() for item in self.items.all()]
         return values
 
     def duplicate(self, label=None, ishtaruser=None):
@@ -5552,11 +5609,13 @@ class Document(BaseHistorizedItem, QRCodeItem, OwnPerms, ImageModel,
             return ""
         return self.image.path
 
-    def get_values(self, prefix="", no_values=False):
-        values = super(Document, self).get_values(prefix=prefix,
-                                                  no_values=no_values)
-        values[prefix + "image_path"] = self.image_path
-        values[prefix + "thumbnail_path"] = self.thumbnail_path
+    def get_values(self, prefix="", no_values=False, filtr=None, **kwargs):
+        values = super(Document, self).get_values(
+            prefix=prefix, no_values=no_values, filtr=filtr, **kwargs)
+        if not filtr or prefix + "image_path" in filtr:
+            values[prefix + "image_path"] = self.image_path
+        if not filtr or prefix + "thumbnail_path" in filtr:
+            values[prefix + "thumbnail_path"] = self.thumbnail_path
         return values
 
     @property
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index 5712a1936..42ab7f9ea 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -48,6 +48,10 @@ from ishtar_common.data_importer import Importer, ImportFormater, \
     ImporterError
 from ishtar_common.utils import task
 
+UnoCalc = None
+if settings.USE_LIBREOFFICE:
+    from ishtar_common.libreoffice import UnoCalc
+
 logger = logging.getLogger(__name__)
 
 
@@ -127,9 +131,8 @@ class ImporterType(models.Model):
         return self.name
 
     def get_libreoffice_template(self):
-        if not settings.USE_LIBREOFFICE:
+        if not UnoCalc:
             return
-        from ishtar_common.libreoffice import UnoCalc
         ROW_NUMBER = 500
         uno = UnoCalc()
         calc = uno.create_calc()