✨ Imports groups: archive_required field - automatically create related imports on group creation - adapt import list

6 files changed, 120 insertions, 20 deletions

diff --git a/ishtar_common/forms_common.py b/ishtar_common/forms_common.py
index 6a351ab3c..d2011338b 100644
--- a/ishtar_common/forms_common.py
+++ b/ishtar_common/forms_common.py
@@ -250,6 +250,7 @@ class BaseImportForm(IshtarForm, forms.ModelForm):
         self.fields["importer_type"].choices = [("", "--")] + [
             (imp.pk, imp.name)
             for imp in models.ImporterType.objects.filter(available=True,
+                                                          is_import=True,
                                                           type=self.importer_type)
         ]
 
@@ -309,9 +310,20 @@ class BaseImportForm(IshtarForm, forms.ModelForm):
         return data
 
 
+def get_archive_from_link(archive_link):
+    request = requests.get(archive_link, stream=True)
+    ntf = tempfile.NamedTemporaryFile()
+    for block in request.iter_content(1024 * 8):
+        if not block:
+            break
+        ntf.write(block)
+    file_name = archive_link.split("/")[-1]
+    return file_name, ntf
+
+
 class NewImportForm(BaseImportForm):
     imported_images_link = forms.URLField(
-        label=_("Associated images (web link to a zip file)"), required=False
+        label=_("Associated documents (web link to a zip file)"), required=False
     )
 
     class Meta:
@@ -334,6 +346,10 @@ class NewImportForm(BaseImportForm):
         "imported_images": FormHeader(_("Documents/Images")),
     }
 
+    def _need_archive(self, data):
+        tpe = data["importer_type"]
+        return tpe.archive_required
+
     def clean(self):
         data = super().clean()
         if data.get("imported_images_link", None) and data.get("imported_images", None):
@@ -353,6 +369,12 @@ class NewImportForm(BaseImportForm):
                     _("\"Associated images\" field must be a valid zip file.")
                 )
         self._clean_csv(is_csv=True)
+        archive_required = self._need_archive(data)
+        if archive_required and (
+                not data.get("imported_images", None) and
+                not data.get("imported_images_link", None)
+        ):
+            raise forms.ValidationError(_("This importer need a document archive."))
         return data
 
     def clean_imported_images_link(self):
@@ -377,14 +399,8 @@ class NewImportForm(BaseImportForm):
         item = super().save(commit)
         if not imported_images_link:
             return item
-        request = requests.get(imported_images_link, stream=True)
-        ntf = tempfile.NamedTemporaryFile()
-        for block in request.iter_content(1024 * 8):
-            if not block:
-                break
-            ntf.write(block)
-        file_name = imported_images_link.split("/")[-1]
-        item.imported_images.save(file_name, File(ntf))
+        file_name, temp_file = get_archive_from_link(imported_images_link)
+        item.imported_images.save(file_name, File(temp_file))
         return item
 
 
@@ -482,6 +498,10 @@ class NewImportGroupForm(NewImportForm):
     def _filter_group(self, user):
         pass
 
+    def _need_archive(self, data):
+        tpe = data["importer_type"]
+        return [sub.importer_type.archive_required for sub in tpe.importer_types.all()]
+
 
 class TargetKeyForm(forms.ModelForm):
     class Meta:
diff --git a/ishtar_common/migrations/0230_auto_20230807_1105.py b/ishtar_common/migrations/0230_auto_20230809_1149.py
index 9d462f393..aaf97671a 100644
--- a/ishtar_common/migrations/0230_auto_20230807_1105.py
+++ b/ishtar_common/migrations/0230_auto_20230809_1149.py
@@ -1,4 +1,4 @@
-# Generated by Django 2.2.24 on 2023-08-07 11:05
+# Generated by Django 2.2.24 on 2023-08-09 11:49
 
 import django.core.validators
 from django.db import migrations, models
@@ -33,6 +33,11 @@ class Migration(migrations.Migration):
         ),
         migrations.AddField(
             model_name='importertype',
+            name='archive_required',
+            field=models.BooleanField(default=False, verbose_name='Archive required'),
+        ),
+        migrations.AddField(
+            model_name='importertype',
             name='is_import',
             field=models.BooleanField(default=True, verbose_name='Can be import'),
         ),
@@ -68,6 +73,11 @@ class Migration(migrations.Migration):
                 'verbose_name_plural': 'Import - Groups',
             },
         ),
+        migrations.AddField(
+            model_name='import',
+            name='group',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='imports', to='ishtar_common.ImportGroup', verbose_name='Group'),
+        ),
         migrations.CreateModel(
             name='ImporterGroupImporter',
             fields=[
@@ -77,12 +87,9 @@ class Migration(migrations.Migration):
                 ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='ishtar_common.ImporterType')),
             ],
             options={
+                'verbose_name': 'Importer - Group <-> Importer',
                 'ordering': ('group', 'order'),
+                'unique_together': {('group', 'order')},
             },
         ),
-        migrations.AddField(
-            model_name='import',
-            name='group',
-            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ImportGroup', verbose_name='Group'),
-        ),
     ]
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index 607184e41..68b571af4 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -216,7 +216,6 @@ __all__ = [
     "DocumentItem",
     "CachedGen",
     "StatisticItem",
-    "CascasdeUpdate",
     "Department",
     "State",
     "CompleteIdentifierItem",
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index 76148a1a6..735ad0b10 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -184,6 +184,7 @@ class ImporterType(models.Model):
     )
     is_template = models.BooleanField(_("Can be exported"), default=False)
     is_import = models.BooleanField(_("Can be import"), default=True)
+    archive_required = models.BooleanField(_("Archive required"), default=False)
     unicity_keys = models.CharField(
         _('Unicity keys (separator ";")'), blank=True, null=True, max_length=500,
         help_text=_("Mandatory for update importer. Set to key that identify items "
@@ -209,7 +210,7 @@ class ImporterType(models.Model):
     @property
     def type_label(self):
         if self.type in IMPORT_TYPES_DICT:
-            return IMPORT_TYPES_DICT[self.type]
+            return IMPORT_TYPES_DICT[str(self.type)]
         return ""
 
     def get_libreoffice_template(self):
@@ -416,6 +417,8 @@ class ImporterGroupImporter(models.Model):
 
     class Meta:
         ordering = ("group", "order")
+        unique_together = ("group", "order")
+        verbose_name = _("Importer - Group <-> Importer")
 
 
 def get_associated_model(parent_model, keys):
@@ -1255,7 +1258,7 @@ class BaseImport(models.Model):
         null=True,
     )
     imported_images = models.FileField(
-        _("Associated images (zip file)"),
+        _("Associated documents (zip file)"),
         upload_to="upload/imports/%Y/%m/",
         blank=True,
         null=True,
@@ -1305,6 +1308,9 @@ class ImportGroup(BaseImport):
         verbose_name_plural = _("Import - Groups")
     ADMIN_SECTION = _("Imports")
 
+    def __str__(self):
+        return f"{self.name} ({self.importer_type.name})"
+
     @property
     def status(self):
         if self.state not in IMPORT_GROUP_STATE_DCT:
@@ -1326,7 +1332,7 @@ class ImportGroup(BaseImport):
             actions.append(("I", _("Re-import")))
             actions.append(("AC", _("Archive")))
         if self.state == "AC":
-            state = "FE" if self.error_file else "F"
+            state = "FE" if any([1 for imp in self.imports.all() if imp.error_file]) else "F"
             actions.append((state, _("Unarchive")))
         if self.state in ("C", "A"):
             actions.append(("ED", _("Edit")))
@@ -1334,7 +1340,32 @@ class ImportGroup(BaseImport):
         return actions
 
     def save(self, *args, **kwargs):
+        add = self._state.adding
         super().save(*args, **kwargs)
+        if not add:
+            return
+        name = f"{self.name} ({self.importer_type.name})"
+        for import_type_relation in self.importer_type.importer_types.all():
+            import_type = import_type_relation.importer_type
+            imp = Import.objects.create(
+                name=name,
+                importer_type=import_type,
+                group=self
+            )
+            modified = False
+            # TODO: only get the relevant sheet
+            if self.imported_file:
+                imported_file = ContentFile(self.imported_file.read())
+                imported_file.name = self.imported_file.name
+                imp.imported_file = imported_file
+                modified = True
+            if import_type.archive_required and self.imported_images:
+                imported_image = ContentFile(self.imported_images.read())
+                imported_image.name = self.imported_images.name
+                imp.imported_images = imported_image
+                modified = True
+            if modified:
+                imp.save()
 
 
 class Import(BaseImport):
diff --git a/ishtar_common/templates/ishtar/import_table.html b/ishtar_common/templates/ishtar/import_table.html
index aec59babc..cf33341d1 100644
--- a/ishtar_common/templates/ishtar/import_table.html
+++ b/ishtar_common/templates/ishtar/import_table.html
@@ -99,6 +99,49 @@ $("#import-list").find('input').prop('disabled', true);
             </td>
         </tr>
         {% endif %}
+        {% if not import.importer_type.type_label %} {# group #}
+        {% for sub in import.imports.all %}
+        <tr>
+            <td></td>
+            <td>
+                {{sub.importer_type}}
+            </td>
+            <td>
+                {% if sub.imported_file %}
+                <a href='{{sub.imported_file.url}}'>{% trans "Source file" %}</a>
+                {% endif %}
+            </td>
+            <td>
+                {% if sub.imported_images %}
+                <a href="{{ sub.imported_images.url }}">{% trans "Media file" %}</a>
+                {% else %}
+                &ndash;
+                {% endif %}
+            </td>
+            <td>
+            </td>
+            <td>
+                {{sub.status}}
+            </td>
+            <td>
+            </td>
+            <td>
+                {% if sub.need_matching %}
+                <a href='{% url "import_link_unmatched" sub.pk %}'>{% trans "Match"%}</a>
+                {% endif %}
+            </td>
+            <td style="white-space: nowrap;">{% if sub.error_file %}
+                <i class="text-danger fa fa-exclamation-triangle" aria-hidden="true"></i> <a href='{{sub.error_file.url}}'>{% trans "File" context "not a directory" %}</a>
+                {% endif %}</td>
+            <td>{% if sub.result_file %}
+                <a href='{{sub.result_file.url}}'>{% trans "File" context "not a directory" %}</a>
+                {% endif %}</td>
+            <td>{% if sub.match_file %}
+                <a href='{{sub.match_file.url}}'>{% trans "File" context "not a directory" %}</a>
+                {% endif %}</td>
+        </tr>
+        {% endfor %}
+        {% endif %}
         {% endfor %}
     </table>
     {% endif %}
diff --git a/ishtar_common/views.py b/ishtar_common/views.py
index 2beade02d..2965ada87 100644
--- a/ishtar_common/views.py
+++ b/ishtar_common/views.py
@@ -1533,7 +1533,7 @@ class ImportListView(IshtarMixin, LoginRequiredMixin, ListView):
             user = models.IshtarUser.objects.get(pk=self.request.user.pk)
             q1 = q1.filter(user=user)
             q2 = q2.filter(user=user)
-        q1 = q1.order_by("-creation_date", "-pk")
+        q1 = q1.filter(group__isnull=True).order_by("-creation_date", "-pk")
         q2 = q2.order_by("-creation_date", "-pk")
         return reversed(sorted(list(q1) + list(q2), key=lambda x: x.creation_date))