Improve document generation perf

7 files changed, 125 insertions, 36 deletions

diff --git a/archaeological_files/models.py b/archaeological_files/models.py
index 8f27f7ec2..6478dfeaa 100644
--- a/archaeological_files/models.py
+++ b/archaeological_files/models.py
@@ -372,7 +372,7 @@ class File(ClosedItem, DocumentItem, BaseHistorizedItem, CompleteIdentifierItem,
     imported_line = models.TextField(_("Imported line"), blank=True, default="")
     history = HistoricalRecords(bases=[HistoryModel])
 
-    GET_VALUES_EXTRA = [
+    GET_VALUES_EXTRA = ValueGetter.GET_VALUES_EXCLUDE_FIELDS + [
         'general_contractor_address_1',
         'general_contractor_address_2',
         'general_contractor_address_3',
diff --git a/archaeological_finds/models_finds.py b/archaeological_finds/models_finds.py
index c7065166e..bab3dae9f 100644
--- a/archaeological_finds/models_finds.py
+++ b/archaeological_finds/models_finds.py
@@ -712,11 +712,15 @@ class FindBasket(Basket, MainItem, ValueGetter):
         )
 
     def get_values(self, prefix='', no_values=False, filtr=None, **kwargs):
+        base_exclude = kwargs["exclude"][:] if "exclude" in kwargs else []
+        base_exclude.append(prefix + "items")
+        kw = kwargs.copy()
+        kw["exclude"] = base_exclude
         values = super(FindBasket, self).get_values(
-            prefix=prefix, no_values=no_values, filtr=filtr, **kwargs)
+            prefix=prefix, no_values=no_values, filtr=filtr, **kw)
         if not filtr or prefix + "items" in filtr:
             values[prefix + "items"] = [
-                item.get_values(no_values=True, filtr=None, **kwargs)
+                item.get_values(no_values=True, filtr=filtr, **kwargs)
                 for item in self.items.distinct().all()
             ]
         return values
@@ -1529,11 +1533,11 @@ class Find(BulkUpdatedItem, ValueGetter, DocumentItem, BaseHistorizedItem,
         'remarkabilities', 'communicabilities', 'preservation_to_considers',
         'alterations', 'alteration_causes', "cultural_attributions"
     ]
-    GET_VALUES_EXTRA_TYPES = (
+    GET_VALUES_EXTRA_TYPES = ValueGetter.GET_VALUES_EXCLUDE_FIELDS + [
         'material_types', 'object_types', 'integrities',
         'remarkabilities', 'communicabilities', 'preservation_to_considers',
         'alterations', 'alteration_causes'
-    )
+    ]
     CACHED_LABELS = ['cached_label', 'cached_periods',
                      'cached_object_types', 'cached_materials']
     objects = UUIDModelManager()
diff --git a/archaeological_finds/tests.py b/archaeological_finds/tests.py
index 2ef551253..bede89b51 100644
--- a/archaeological_finds/tests.py
+++ b/archaeological_finds/tests.py
@@ -49,8 +49,7 @@ from archaeological_context_records.models import Period, Dating, \
     ContextRecord, DatingType, DatingQuality
 from archaeological_finds import models, views
 from archaeological_warehouse.models import Warehouse, WarehouseType, \
-    ContainerType, Container, WarehouseDivision, WarehouseDivisionLink, \
-    ContainerLocalisation
+    ContainerType, Container, WarehouseDivisionLink
 from archaeological_operations.models import Operation, OperationType
 
 from ishtar_common import forms_common
@@ -2579,3 +2578,67 @@ class LabelTest(FindInit, TestCase):
                 self.templates.append(doc.template.path)
             else:
                 self.assertFalse(doc.template.name, msg=msg)
+
+
+class TemplateTest(FindInit, TestCase):
+    fixtures = FIND_FIXTURES
+    model = models.Find
+
+    def setUp(self):
+        template = settings.ROOT_PATH + \
+                   '../archaeological_finds/tests/notices-panier.odt'
+        filename = template.split("/")[-1]
+        shutil.copy(template,
+                    os.path.join(settings.MEDIA_ROOT, filename),
+                    follow_symlinks=True)
+        self.template = os.path.join(settings.MEDIA_ROOT, filename)
+        self.templates = [self.template]
+        ope1 = self.create_operation()[0]
+        ope2 = self.create_operation()[1]
+        cr = self.create_context_record(
+            data={"label": "CR 1", "operation": ope1}
+        )[0]
+        cr2 = self.create_context_record(
+            data={"label": "CR 2", "operation": ope2}
+        )[1]
+        self.create_finds(data_base={"context_record": cr})[0]
+        self.create_finds(data_base={"context_record": cr2})[1]
+        self.basket = models.FindBasket.objects.create(label="Hophop")
+        self.basket.items.add(self.finds[0])
+        self.basket.items.add(self.finds[1])
+        wt, __ = WarehouseType.objects.get_or_create(label="WT", txt_idx="WT")
+        location = Warehouse.objects.create(name="Warehouse", warehouse_type=wt)
+        ct, __ = ContainerType.objects.get_or_create(label="CT", txt_idx="CT")
+        self.container = Container.objects.create(
+            location=location,
+            reference="Reférence",
+            container_type=ct
+        )
+
+    def test_label(self):
+        with open(self.template, 'rb') as tpl:
+            template = SimpleUploadedFile("template.odt",
+                                          tpl.read())
+            model, __ = ImporterModel.objects.get_or_create(
+                klass='archaeological_finds.models.Find'
+            )
+            q = DocumentTemplate.objects.filter(slug="test")
+            if q.count():
+                q.all()[0].delete()
+            doc = DocumentTemplate.objects.create(
+                name="Test",
+                slug="test",
+                associated_model=model,
+                available=True,
+                template=template)
+        self.templates.append(doc.template.path)
+        doc = DocumentTemplate.objects.get(pk=doc.pk)
+
+        result = doc.publish(self.basket)
+        if result:
+            self.templates.append(result)
+
+    def tearDown(self):
+        for tpl in self.templates:
+            if os.path.exists(tpl):
+                os.remove(tpl)
diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py
index 287b01013..d8c61c642 100644
--- a/archaeological_operations/tests.py
+++ b/archaeological_operations/tests.py
@@ -2562,7 +2562,7 @@ class RegisterTest(TestCase, OperationInitTest):
         self.act_types, self.admin_acts = create_administrativact(
             self.user, self.operations[0])
 
-    def testSearch(self):
+    def test_search(self):
         c = Client()
         response = c.get(reverse('get-administrativeact'), {'year': '2014'})
         # no result when no authentication
diff --git a/archaeological_operations/tests/document_reference.odt b/archaeological_operations/tests/document_reference.odt
index 0097e8b15..c1b43053f 100755
--- a/archaeological_operations/tests/document_reference.odt
+++ b/archaeological_operations/tests/document_reference.odt
diff --git a/archaeological_warehouse/models.py b/archaeological_warehouse/models.py
index 06c7cdcd8..cb144270b 100644
--- a/archaeological_warehouse/models.py
+++ b/archaeological_warehouse/models.py
@@ -651,6 +651,9 @@ class Container(DocumentItem, Merge, LightHistorizedItem,
     STATISTIC_MODALITIES = [
         key for key, lbl in STATISTIC_MODALITIES_OPTIONS.items()]
 
+    GET_VALUES_EXCLUDE_FIELDS = ValueGetter.GET_VALUES_EXCLUDE_FIELDS + [
+        "inside_container", "parent"]
+
     # search parameters
     EXTRA_REQUEST_KEYS = {
         'location': 'location__pk',
@@ -1341,16 +1344,17 @@ class Container(DocumentItem, Merge, LightHistorizedItem,
     def get_values(self, prefix='', no_values=False, filtr=None, **kwargs):
         values = super(Container, self).get_values(
             prefix=prefix, no_values=no_values, filtr=filtr, **kwargs)
-        if (not filtr or prefix + 'finds' in filtr) and (
-                not prefix.startswith("container_") and not
-                prefix.startswith("container_ref_")):
+        from_find = prefix.startswith("container_") or \
+                    prefix.startswith("container_ref_")
+        if (not filtr or prefix + 'finds' in filtr) and not from_find:
             kwargs["exclude"] = [prefix + "container", prefix + "container_ref"]
             # prevent recursive call
             values[prefix + 'finds'] = [
                 f.get_values(
                     prefix=prefix, no_values=True, filtr=None, **kwargs)
                 for f in self.finds.distinct().all()]
-        if (not filtr or prefix + 'operation' in filtr or
+        if not from_find and (
+                not filtr or prefix + 'operation' in filtr or
                 prefix + "context_record" in filtr) and self.finds.count():
             # assume that only one operation is in this container
             # you should know what you are doing when using theses variables
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index bf8c8c19c..2df42bc05 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -174,16 +174,17 @@ class ValueGetter(object):
                 value = getattr(self, field_name)
             except (AttributeError, MultipleObjectsReturned):
                 continue
-            if field_name in self.GET_VALUES_EXCLUDE_FIELDS:
+            if field_name in self.GET_VALUES_EXCLUDE_FIELDS or \
+                    prefix + field_name in exclude:
                 continue
-            if prefix + field_name in exclude:
+            if filtr and not any(
+                    field_name for f in filtr
+                    if f.startswith(prefix + field_name)):
                 continue
             if hasattr(value, 'get_values'):
                 new_prefix = prefix + field_name + '_'
                 values.update(
                     value.get_values(new_prefix, filtr=filtr, **kwargs))
-            if filtr and prefix + field_name not in filtr:
-                continue
             if hasattr(self, "get_values_for_" + field_name):
                 values[prefix + field_name] = getattr(
                     self, "get_values_for_" + field_name)()
@@ -1575,19 +1576,22 @@ class DocumentTemplate(models.Model):
                       slugify(self.name.replace(' ', '_').lower()) + '-' + \
                       datetime.date.today().strftime('%Y-%m-%d') + \
                       "." + self.template.name.split('.')[-1]
-        """
-        # to be tested
+        item_re = r"([A-Za-z0-9_.]*)(\|[A-Za-z0-9_.-]*)*[|]*"
         filtr = self.get_filter(
             self.template,
-            [r'{{ *([A-Za-z0-9_.]*)(\|[A-Za-z0-9_.-]*)* *}}',
-             r'{% *for +[A-Za-z0-9_]+ +in +'
-             r'([A-Za-z0-9_.]*)(\|[A-Za-z0-9_.-]*)* *%}',
-             r'{% *if +([A-Za-z0-9_.]*)(\|[A-Za-z0-9_.-]*)* *%}',
-             ]
+            [
+                # {{ key1.key2 }}
+                r'{{ *' + item_re + ' *}}',
+                # {% for item in key1.key2 %}
+                r'{% *for +[A-Za-z0-9_]+ +in +' + item_re + r' *%}',
+                # {% if ** %}
+                r'{% (?:el)*if ([^}]*)%}',
+            ]
         )
-        """
-        filtr = []
-        values = c_object.get_values()
+        # values = c_object.get_values(filtr=[])
+        if "VALUES" in filtr:
+            filtr = []
+        values = c_object.get_values(filtr=filtr)
         if not filtr or 'VALUES' in filtr:
             values['VALUES'] = json.dumps(
                 values, indent=4, sort_keys=True,
@@ -1624,16 +1628,37 @@ class DocumentTemplate(models.Model):
                     filtr.append(key)
         new_filter = []
         for fil in filtr:
-            keys = fil.split('.')
+            if not fil:
+                continue
+            new_filter += [f for f in fil.split(" ")
+                           if f and f not in (
+                               "==", "not", "in", ">", "<", "!=",
+                               ">", "<", ">=", "<=", "or", "&gt;=",
+                               "&lt;="
+                           )]
+        filtr = new_filter
+        new_filter = []
+        for fil in filtr:
+            if not fil or fil in ("in", "not", "el") or fil.startswith("|"):
+                continue
+            try:
+                int(fil)
+                continue
+            except ValueError:  # not a single int
+                pass
+            keys = fil.split("|")[0].split('.')
+            new_filter += keys
             prefix = ""
             for k in keys:
+                if not k:
+                    continue
                 if prefix:
                     prefix += '_'
                 if prefix + k in new_filter:
                     continue
                 new_filter.append(prefix + k)
                 prefix += k
-        return new_filter
+        return list(set(new_filter))
 
     def publish_labels(self, objects):
         if not objects:
@@ -1653,7 +1678,7 @@ class DocumentTemplate(models.Model):
                 # {{items.4.key}}
                 r'{{ *' + item_re + r' *}}',
                 # {% if items.42.other_key %}
-                r'{% *if ' + item_re + r' *%}',
+                r'{% *(el)*if ' + item_re + r' *%}',
                 # {% for item in items.42.another_keys %}
                 r'{% *for +[A-Za-z0-9_]+ +in +' + item_re + r' *%}',
              ]
@@ -2666,13 +2691,6 @@ class Basket(FullSearch, OwnPerms, ValueGetter, TemplateItem):
         return Q(user=ishtaruser) | Q(
             shared_write_with=ishtaruser)
 
-    def get_values(self, prefix='', no_values=False, filtr=None, **kwargs):
-        values = super(Basket, self).get_values(
-            prefix=prefix, no_values=no_values, filtr=filtr, **kwargs)
-        if not filtr or "items" in filtr:
-            values["items"] = [item.get_values() for item in self.items.all()]
-        return values
-
     def duplicate(self, label=None, ishtaruser=None):
         """
         Duplicate the basket. Items in basket are copied but not shared users