Zip/unzip files on archive/unarchive imports

5 files changed, 229 insertions, 3 deletions

diff --git a/ishtar_common/migrations/0212_auto_20210219_1408.py b/ishtar_common/migrations/0212_auto_20210219_1408.py
new file mode 100644
index 000000000..25d49e75f
--- /dev/null
+++ b/ishtar_common/migrations/0212_auto_20210219_1408.py
@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.27 on 2021-02-19 14:08
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import ishtar_common.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ishtar_common', '0211_auto_20210111_1321'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='import',
+            name='archive_file',
+            field=models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=255, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Archive file'),
+        ),
+        migrations.AddField(
+            model_name='ishtarsiteprofile',
+            name='delete_image_zip_on_archive',
+            field=models.BooleanField(default=False, verbose_name='Import - Delete image/document zip on archive'),
+        ),
+        migrations.AlterField(
+            model_name='historicalorganization',
+            name='grammatical_gender',
+            field=models.CharField(blank=True, choices=[('M', 'Male'), ('F', 'Female'), ('N', 'Neutral')], default='', help_text=ishtar_common.models.documentation_get_gender_values, max_length=1, verbose_name='Grammatical gender'),
+        ),
+        migrations.AlterField(
+            model_name='organization',
+            name='grammatical_gender',
+            field=models.CharField(blank=True, choices=[('M', 'Male'), ('F', 'Female'), ('N', 'Neutral')], default='', help_text=ishtar_common.models.documentation_get_gender_values, max_length=1, verbose_name='Grammatical gender'),
+        ),
+        migrations.AlterField(
+            model_name='organizationtype',
+            name='grammatical_gender',
+            field=models.CharField(blank=True, choices=[('M', 'Male'), ('F', 'Female'), ('N', 'Neutral')], default='', help_text=ishtar_common.models.documentation_get_gender_values, max_length=1, verbose_name='Grammatical gender'),
+        ),
+        migrations.AlterField(
+            model_name='titletype',
+            name='grammatical_gender',
+            field=models.CharField(blank=True, choices=[('M', 'Male'), ('F', 'Female'), ('N', 'Neutral')], default='', help_text=ishtar_common.models.documentation_get_gender_values, max_length=1, verbose_name='Grammatical gender'),
+        ),
+    ]
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index 8e65c4503..0a2f4bb6f 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -737,6 +737,9 @@ class IshtarSiteProfile(models.Model, Cached):
     warning_name = models.TextField(_("Warning name"), blank=True, default="")
     warning_message = models.TextField(_("Warning message"), blank=True,
                                        default="")
+    delete_image_zip_on_archive = models.BooleanField(
+        _("Import - Delete image/document zip on archive"), default=False
+    )
     config = models.CharField(
         _("Alternate configuration"), max_length=200,
         choices=ALTERNATE_CONFIGS_CHOICES,
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index 7a8a10bc2..5a3af1a05 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -19,6 +19,7 @@
 
 import csv
 import datetime
+import json
 import os
 import logging
 import shutil
@@ -26,9 +27,11 @@ import re
 import tempfile
 import zipfile
 
+from django.apps import apps
 from django.conf import settings
 from django.contrib.gis.db import models
 from django.core.exceptions import ValidationError
+from django.core.files import File
 from django.core.files.base import ContentFile
 from django.core.validators import validate_comma_separated_integer_list
 from django.db.models.base import ModelBase
@@ -965,6 +968,9 @@ class Import(models.Model):
     match_file = models.FileField(
         _("Match file"), upload_to="upload/imports/%Y/%m/", blank=True,
         null=True, max_length=255, help_text=max_size_help())
+    archive_file = models.FileField(
+        _("Archive file"), upload_to="upload/imports/%Y/%m/", blank=True,
+        null=True, max_length=255, help_text=max_size_help())
     state = models.CharField(_("State"), max_length=2, choices=IMPORT_STATE,
                              default='C')
     conservative_import = models.BooleanField(
@@ -1096,7 +1102,7 @@ class Import(models.Model):
         """
         Get available action relevant with the current status
         """
-        from ishtar_common.models import IshtarSiteProfile
+        IshtarSiteProfile = apps.get_model("ishtar_common", "IshtarSiteProfile")
         profile = IshtarSiteProfile.get_current_profile()
         actions = []
         if self.state == 'C':
@@ -1121,7 +1127,8 @@ class Import(models.Model):
                     actions.append(('CH', _("Check for changes")))
             actions.append(('AC', _("Archive")))
         if self.state == 'AC':
-            actions.append(('A', _("Unarchive")))
+            state = "FE" if self.error_file else "F"
+            actions.append((state, _("Unarchive")))
         actions.append(('D', _("Delete")))
         return actions
 
@@ -1357,10 +1364,94 @@ class Import(models.Model):
         if return_importer_and_data:
             return importer, data
 
+    def _unarchive(self):
+        if not self.archive_file:
+            return
+        with tempfile.TemporaryDirectory() as tmp_dir_name:
+            # extract the current archive
+            current_zip = zipfile.ZipFile(self.archive_file.path, 'r')
+            name_list = current_zip.namelist()
+            if "content.json" not in name_list:
+                return
+            for name in name_list:
+                current_zip.extract(name, tmp_dir_name)
+            current_zip.close()
+            content_name = os.path.join(tmp_dir_name, "content.json")
+            try:
+                with open(content_name, "r") as content:
+                    files = json.loads(content.read())
+            except (IOError, json.JSONDecodeError):
+                return
+            today = datetime.date.today()
+            for attr in files:
+                filename = files[attr]
+                full_filename = os.path.join(tmp_dir_name, filename)
+                with open(full_filename, "rb") as raw_file:
+                    getattr(self, attr).save(
+                        "upload/imports/{}/{:02d}/{}".format(
+                            today.year, today.month, filename),
+                        File(raw_file)
+                    )
+
+        os.remove(self.archive_file.path)
+        setattr(self, 'archive_file', None)
+        self.state = "FE" if self.error_file else "F"
+        self.save()
+        return True
+
+    def _archive(self):
+        file_attr = ["imported_file", "error_file", "result_file",
+                     "match_file"]
+        files = [
+            (k, getattr(self, k).path, getattr(self, k).name.split(os.sep)[-1])
+            for k in file_attr
+            if getattr(self, k)
+        ]
+        self._archive_pending = True
+        with tempfile.TemporaryDirectory() as tmpdir:
+            base_name = "{}.zip".format(slugify(self.name))
+            archive_name = os.path.join(tmpdir, base_name)
+            with zipfile.ZipFile(archive_name, "w") as current_zip:
+                zip_content = {}
+                for k, path, name in files:
+                    try:
+                        current_zip.write(path, arcname=name)
+                        zip_content[k] = name
+                    except OSError:
+                        pass
+                content_name = os.path.join(tmpdir, "content.json")
+                with open(content_name, "w") as content:
+                    content.write(json.dumps(zip_content))
+                current_zip.write(content_name, arcname="content.json")
+
+            today = datetime.date.today()
+            with open(archive_name, "rb", ) as raw_file:
+                self.archive_file.save(
+                    "upload/imports/{}/{:02d}/{}".format(
+                        today.year, today.month, base_name),
+                    File(raw_file)
+                )
+        IshtarSiteProfile = apps.get_model("ishtar_common", "IshtarSiteProfile")
+        profile = IshtarSiteProfile.get_current_profile()
+        if profile.delete_image_zip_on_archive:
+            file_attr.append("imported_images")
+        for attr in file_attr:
+            file_field = getattr(self, attr)
+            if file_field:
+                os.remove(file_field.path)
+                setattr(self, attr, None)
+        self.save()
+        self._archive_pending = False
+
     def archive(self):
         self.state = 'AC'
         self.end_date = datetime.datetime.now()
-        self.save()
+        self._archive()
+
+    def unarchive(self, state):
+        if not self._unarchive():
+            self.state = state
+            self.save()  # only save if no save previously
 
     def get_all_imported(self):
         imported = []
@@ -1370,6 +1461,12 @@ class Import(models.Model):
                          for obj in getattr(self, accessor).all()]
         return imported
 
+    def save(self, *args, **kwargs):
+        super(Import, self).save(*args, **kwargs)
+        if self.state == "AC" and not getattr(
+                self, "_archive_pending", False) and not self.archive_file:
+            self._archive()
+
 
 def pre_delete_import(sender, **kwargs):
     # deleted imported items when an import is delete
diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py
index d6802ef77..71e599d59 100644
--- a/ishtar_common/tests.py
+++ b/ishtar_common/tests.py
@@ -2274,6 +2274,84 @@ class ImportTest(TestCase):
                 imported_file=mcc_operation_file)
         return imprt
 
+    def test_archive_import(self):
+        imprt = self.create_import()
+        with open(imprt.imported_file.path, "r") as f:
+            csv_content = f.read()
+        with tempfile.TemporaryDirectory() as tmpdir:
+            for k in ("error_file", "result_file", "match_file",
+                      "imported_images"):
+                sample_file = os.path.join(tmpdir, "media_{}.zip".format(k))
+                with open(sample_file, "w") as m:
+                    m.write("test" + k)
+                with open(sample_file, "rb") as raw_file:
+                    getattr(imprt, k).save("media.txt", DjangoFile(raw_file))
+        profile = models.get_current_profile()
+        profile.delete_image_zip_on_archive = False
+        profile.save()
+        imprt.archive()
+        imprt = models.Import.objects.get(pk=imprt.pk)
+        self.assertEqual(imprt.state, "AC")
+        self.assertFalse(imprt.error_file)
+        self.assertFalse(imprt.result_file)
+        self.assertFalse(imprt.match_file)
+        self.assertTrue(imprt.imported_images)
+        self.assertTrue(imprt.archive_file)
+        self.assertTrue(zipfile.is_zipfile(imprt.archive_file))
+        with tempfile.TemporaryDirectory() as tmpdir:
+            current_zip = zipfile.ZipFile(imprt.archive_file.path, 'r')
+            name_list = current_zip.namelist()
+            self.assertIn("content.json", name_list)
+            current_zip.extract("content.json", tmpdir)
+            content_name = os.path.join(tmpdir, "content.json")
+            with open(content_name, "r") as content:
+                files = json.loads(content.read())
+            self.assertIn("imported_file", files.keys())
+            self.assertIn(files["imported_file"], name_list)
+            self.assertIn("error_file", files.keys())
+            self.assertIn(files["error_file"], name_list)
+            self.assertIn("result_file", files.keys())
+            self.assertIn(files["result_file"], name_list)
+            self.assertIn("match_file", files.keys())
+            self.assertIn(files["match_file"], name_list)
+            rev_dict = {v: k for k, v in files.items()}
+            for name in name_list:
+                current_zip.extract(name, tmpdir)
+                if name.endswith(".txt"):
+                    with open(os.path.join(tmpdir, name), "r") as f:
+                        self.assertEqual(f.read(), "test" + rev_dict[name])
+                elif name.endswith(".csv"):  # imported file
+                    with open(os.path.join(tmpdir, name), "r") as f:
+                        self.assertEqual(f.read(), csv_content)
+
+        imprt.unarchive('FE')
+        imprt = models.Import.objects.get(pk=imprt.pk)
+        self.assertEqual(imprt.state, "FE")
+        for k in ("error_file", "result_file", "match_file", "imported_images"):
+            field = getattr(imprt, k)
+            self.assertTrue(field, "{} is missing in unarchive".format(k))
+            with open(field.path, "r") as f:
+                self.assertEqual(f.read(), "test" + k)
+        field = getattr(imprt, "imported_file")
+        self.assertTrue(field, "{} is missing in unarchive".format(k))
+        with open(field.path, "r") as f:
+            self.assertEqual(f.read(), csv_content)
+
+        profile = models.get_current_profile()
+        profile.delete_image_zip_on_archive = True
+        profile.save()
+        imprt = models.Import.objects.get(pk=imprt.pk)
+        image_filename = imprt.imported_images.path
+        self.assertTrue(os.path.isfile(image_filename))
+        imprt.archive()
+        imprt = models.Import.objects.get(pk=imprt.pk)
+        self.assertFalse(imprt.imported_images)
+        self.assertFalse(os.path.isfile(image_filename))
+        imprt.unarchive("F")
+        imprt = models.Import.objects.get(pk=imprt.pk)
+        self.assertEqual(imprt.state, "FE")  # as an error file so state fixed
+        self.assertFalse(imprt.imported_images)
+
     def test_delete_related(self):
         town = models.Town.objects.create(name='my-test')
         self.assertEqual(models.Town.objects.filter(name='my-test').count(), 1)
diff --git a/ishtar_common/views.py b/ishtar_common/views.py
index 874252962..b9cf98a87 100644
--- a/ishtar_common/views.py
+++ b/ishtar_common/views.py
@@ -1315,6 +1315,8 @@ class ImportListView(IshtarMixin, LoginRequiredMixin, ListView):
                 )
             elif action == 'AC':
                 imprt.archive()
+            elif action in ('F', 'FE'):
+                imprt.unarchive(action)
         return HttpResponseRedirect(reverse(self.current_url))
 
     def get_context_data(self, **kwargs):