Fix label generation filter + tests

4 files changed, 120 insertions, 66 deletions

diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index 55fb1566c..977b80ac6 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -1580,46 +1580,18 @@ class DocumentTemplate(models.Model):
     def get_baselink_for_labels(self):
         return reverse('generate-labels', args=[self.slug])
 
-    def publish(self, c_object):
-        tempdir = tempfile.mkdtemp("-ishtardocs")
-        output_name = tempdir + os.path.sep + \
-                      slugify(self.name.replace(' ', '_').lower()) + '-' + \
-                      datetime.date.today().strftime('%Y-%m-%d') + \
-                      "." + self.template.name.split('.')[-1]
-        item_re = r"([A-Za-z0-9_.]*)(\|[A-Za-z0-9_.-]*)*[|]*"
-        filtr = self.get_filter(
-            self.template,
-            [
-                # {{ key1.key2 }}
-                r'{{ *' + item_re + ' *}}',
-                # {% for item in key1.key2 %}
-                r'{% *for +[A-Za-z0-9_]+ +in +' + item_re + r' *%}',
-                # {% if ** %}
-                r'{% (?:el)*if ([^}]*)%}',
-            ]
-        )
-        # values = c_object.get_values(filtr=[])
-        if "VALUES" in filtr:
-            filtr = []
-        values = c_object.get_values(filtr=filtr)
-        if not filtr or 'VALUES' in filtr:
-            values['VALUES'] = json.dumps(
-                values, indent=4, sort_keys=True,
-                skipkeys=True, ensure_ascii=False,
-                separators=("", " : "),
-            ).replace(" " * 4, "\t")
-        engine = IshtarSecretaryRenderer()
+    def _exclude_filter(self, value):
+        """
+        value is excluded from values to fetch?
+        """
+        if not value or value in ("in", "not", "el") or value.startswith("|"):
+            return True
         try:
-            result = engine.render(self.template, **values)
-        except TemplateSyntaxError as e:
-            raise TemplateSyntaxError(str(e), e.lineno)
-        except UndefinedError as e:
-            raise TemplateSyntaxError(str(e), 0)
-        except Exception as e:
-            raise TemplateSyntaxError(str(e), 0)
-        with open(output_name, 'wb') as output:
-            output.write(result)
-        return output_name
+            int(value)
+            return True
+        except ValueError:  # not a single int
+            pass
+        return False
 
     def get_filter(self, template, regexp_list=None):
         if not regexp_list:
@@ -1637,30 +1609,21 @@ class DocumentTemplate(models.Model):
                 if key not in filtr:
                     filtr.append(key)
         new_filter = []
+        OPERATORS = ["==", "not", "in", "&gt;", "&lt;", "!=", ">", "<", ">=",
+                     "<=", "or", "&gt;=", "&lt;="]
         for fil in filtr:
             if not fil:
                 continue
             new_filter += [f for f in fil.split(" ")
-                           if f and f not in (
-                               "==", "not", "in", "&gt;", "&lt;", "!=",
-                               ">", "<", ">=", "<=", "or", "&gt;=",
-                               "&lt;="
-                           )]
+                           if f and f not in OPERATORS]
         filtr = new_filter
         new_filter = []
         for fil in filtr:
-            if not fil or fil in ("in", "not", "el") or fil.startswith("|"):
-                continue
-            try:
-                int(fil)
-                continue
-            except ValueError:  # not a single int
-                pass
-            keys = fil.split("|")[0].split('.')
-            new_filter += keys
+            keys = fil.strip().split("|")[0].split('.')
+            new_filter += [k for k in keys if not self._exclude_filter(k)]
             prefix = ""
             for k in keys:
-                if not k:
+                if self._exclude_filter(k):
                     continue
                 if prefix:
                     prefix += '_'
@@ -1670,6 +1633,56 @@ class DocumentTemplate(models.Model):
                 prefix += k
         return list(set(new_filter))
 
+    ITEM_RE = r"([A-Za-z0-9_.]*)(?:[\[\]0-9-:])*(?:\|[^}]+)*"
+    BASE_RE = [
+        # {{ key1.key2 }}
+        r'{{ *' + ITEM_RE + ' *}}',
+        # {% for item in key1.key2 %}
+        r'{% *for +[A-Za-z0-9_]+ +in +' + ITEM_RE + r' *%}',
+        # {% if ** %}
+        r'{% (?:el)*if ([^}]*)%}',
+        ]
+
+    def publish(self, c_object):
+        tempdir = tempfile.mkdtemp("-ishtardocs")
+        output_name = tempdir + os.path.sep + \
+                      slugify(self.name.replace(' ', '_').lower()) + '-' + \
+                      datetime.date.today().strftime('%Y-%m-%d') + \
+                      "." + self.template.name.split('.')[-1]
+        filtr = self.get_filter(self.template, self.BASE_RE)
+        # values = c_object.get_values(filtr=[])
+        if "VALUES" in filtr:
+            filtr = []
+        values = c_object.get_values(filtr=filtr)
+        if not filtr or 'VALUES' in filtr:
+            values['VALUES'] = json.dumps(
+                values, indent=4, sort_keys=True,
+                skipkeys=True, ensure_ascii=False,
+                separators=("", " : "),
+            ).replace(" " * 4, "\t")
+        engine = IshtarSecretaryRenderer()
+        try:
+            result = engine.render(self.template, **values)
+        except TemplateSyntaxError as e:
+            raise TemplateSyntaxError(str(e), e.lineno)
+        except UndefinedError as e:
+            raise TemplateSyntaxError(str(e), 0)
+        except Exception as e:
+            raise TemplateSyntaxError(str(e), 0)
+        with open(output_name, 'wb') as output:
+            output.write(result)
+        return output_name
+
+    LABEL_ITEM_RE = r"items\.\d\.([A-Za-z0-9_.]*)(?:[\[\]0-9-:])*(?:\|[^}]+)*"
+    LABEL_RE = [
+        # {{items.4.key}}
+        r'{{ *' + LABEL_ITEM_RE + r' *}}',
+        # {% if ** %}
+        r'{% (?:el)*if ([^}]*)%}',
+        # {% for item in items.42.another_keys %}
+        r'{% *for +[A-Za-z0-9_]+ +in +' + LABEL_ITEM_RE + r' *%}',
+    ]
+
     def publish_labels(self, objects):
         if not objects:
             return
@@ -1681,18 +1694,7 @@ class DocumentTemplate(models.Model):
         len_objects = len(objects)
         names = []
 
-        item_re = r"items\.\d\.([A-Za-z0-9_.]*)(?:\|[^}]+)*"
-        filtr = self.get_filter(
-            self.template,
-            [
-                # {{items.4.key}}
-                r'{{ *' + item_re + r' *}}',
-                # {% if items.42.other_key %}
-                r'{% *(el)*if ' + item_re + r' *%}',
-                # {% for item in items.42.another_keys %}
-                r'{% *for +[A-Za-z0-9_]+ +in +' + item_re + r' *%}',
-             ]
-        )
+        filtr = self.get_filter(self.template, self.LABEL_RE)
         for idx in range(int(len(objects) / self.label_per_page) + 1):
             if idx * self.label_per_page >= len_objects:
                 break
diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py
index 72836fa45..b5de13fe1 100644
--- a/ishtar_common/tests.py
+++ b/ishtar_common/tests.py
@@ -2909,3 +2909,55 @@ class JinjaFilterTest(TestCase):
             "sAINT GEORGES D'OLÉRON")
         self.assertEqual(utils_secretary.lowerfirst_filter("S"), "s")
         self.assertEqual(utils_secretary.lowerfirst_filter(""), "")
+
+
+class TemplateGenerationTest(TestCase):
+    def test_filters(self):
+        model, __ = models.ImporterModel.objects.get_or_create(
+            klass='archaeological_finds.models.Find'
+        )
+        q = models.DocumentTemplate.objects.filter(slug="test")
+        if q.count():
+            q.all()[0].delete()
+        doc = models.DocumentTemplate.objects.create(
+            name="Test",
+            slug="test",
+            associated_model=model,
+            available=True)
+
+        LABEL_EXPECTED_KEYS = [
+            'container_index',
+            'context_record_operation_common_name',
+            'context_record_town_name',
+            'container_location_name',
+            'context_record_operation_operation_type',
+            'container_reference',
+            'my_complete_id',
+            'complete_idx',
+            'complete_idxy',
+        ]
+        tpl_label = settings.ROOT_PATH + \
+            '../ishtar_common/tests/test-filters-label.odt'
+        BASE_EXPECTED_KEYS = [
+            'container_index',
+            'context_record_operation_common_name',
+            'context_record_town_name',
+            'container_location_name',
+            'context_record_operation_operation_type',
+            'container_reference',
+            'my_complete_id',
+            'complete_idx',
+            'complete_idxy',
+        ]
+        tpl_base = settings.ROOT_PATH + \
+              '../ishtar_common/tests/test-filters-base.odt'
+        tests = (
+            (LABEL_EXPECTED_KEYS, tpl_label, models.DocumentTemplate.LABEL_RE),
+            (BASE_EXPECTED_KEYS, tpl_base, models.DocumentTemplate.BASE_RE),
+        )
+        for expected_keys, tpl, filter_re in tests:
+            with open(tpl, 'rb') as tpl:
+                template = SimpleUploadedFile("template.odt", tpl.read())
+                filtr = doc.get_filter(template, filter_re)
+                for key in expected_keys:
+                    self.assertIn(key, filtr)
diff --git a/ishtar_common/tests/test-filters-base.odt b/ishtar_common/tests/test-filters-base.odt
new file mode 100644
index 000000000..81b5ffb5c
--- /dev/null
+++ b/ishtar_common/tests/test-filters-base.odt
diff --git a/ishtar_common/tests/test-filters-label.odt b/ishtar_common/tests/test-filters-label.odt
new file mode 100644
index 000000000..03942c641
--- /dev/null
+++ b/ishtar_common/tests/test-filters-label.odt