✨ Import group: archive

1 files changed, 140 insertions, 27 deletions

diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index 9e505392d..b03b42e1a 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -1359,6 +1359,14 @@ class BaseImport(models.Model):
         max_length=220,
         help_text=max_size_help(),
     )
+    archive_file = models.FileField(
+        _("Archive file"),
+        upload_to="upload/imports/%Y/%m/",
+        blank=True,
+        null=True,
+        max_length=255,
+        help_text=max_size_help(),
+    )
     encoding = models.CharField(
         _("Encoding"), choices=ENCODINGS, default="utf-8", max_length=15,
         help_text=_("Only required for CSV file"),
@@ -1384,6 +1392,7 @@ class BaseImport(models.Model):
     end_date = models.DateTimeField(
         _("End date"), auto_now_add=True, blank=True, null=True, editable=False
     )
+    state = None
 
     class Meta:
         abstract = True
@@ -1396,6 +1405,31 @@ class BaseImport(models.Model):
     def pre_import_form_is_valid(self) -> bool:
         raise NotImplemented()
 
+    def _archive(self):
+        raise NotImplemented()
+
+    def _unarchive(self):
+        raise NotImplemented()
+
+    def archive(self):
+        self.state = "AC"
+        self.end_date = datetime.datetime.now()
+        self._archive()
+
+    def unarchive(self, state):
+        if not self._unarchive():
+            self.state = state
+            self.save()  # only save if no save previously
+
+    def save(self, *args, **kwargs):
+        super().save(*args, **kwargs)
+        if (
+                self.state == "AC"
+                and not getattr(self, "_archive_pending", False)
+                and not self.archive_file
+        ):
+            self._archive()
+
 
 class ImportGroup(BaseImport):
     importer_type = models.ForeignKey(ImporterGroup, on_delete=models.CASCADE,
@@ -1508,6 +1542,112 @@ class ImportGroup(BaseImport):
         self.end_date = datetime.datetime.now()
         self.save()
 
+    def _unarchive(self):
+        if not self.archive_file:
+            return
+        with tempfile.TemporaryDirectory() as tmp_dir_name:
+            # extract the current archive
+            current_zip = zipfile.ZipFile(self.archive_file.path, "r")
+            name_list = current_zip.namelist()
+            if "content.json" not in name_list:
+                return
+            for name in name_list:
+                current_zip.extract(name, tmp_dir_name)
+            current_zip.close()
+            content_name = os.path.join(tmp_dir_name, "content.json")
+            try:
+                with open(content_name, "r") as content:
+                    files = json.loads(content.read())
+            except (IOError, json.JSONDecodeError):
+                return
+            today = datetime.date.today()
+            for attr in files:
+                filename = files[attr]
+                full_filename = os.path.join(tmp_dir_name, filename)
+                with open(full_filename, "rb") as raw_file:
+                    getattr(self, attr).save(
+                        "upload/imports/{}/{:02d}/{}".format(
+                            today.year, today.month, filename
+                        ),
+                        File(raw_file),
+                    )
+
+        os.remove(self.archive_file.path)
+        setattr(self, "archive_file", None)
+        self.state = "FE" if self.error_file else "F"
+        self.save()
+        return True
+
+    def _archive(self):
+        file_attr = ["imported_file"]
+        sub_import_file_attr = ["error_file", "result_file", "match_file"]
+        files = [
+            (k, getattr(self, k).path, getattr(self, k).name.split(os.sep)[-1])
+            for k in file_attr
+            if getattr(self, k)
+        ]
+        import_list = self.import_list()
+        for idx, sub_import in enumerate(import_list):
+            files += [
+                (f"sub-{idx}-{k}", getattr(sub_import, k).path,
+                 getattr(sub_import, k).name.split(os.sep)[-1])
+                for k in sub_import_file_attr
+                if getattr(sub_import, k)
+            ]
+        with tempfile.TemporaryDirectory("-ishtar") as tmpdir:
+            base_name = "{}.zip".format(slugify(self.name))
+            archive_name = os.path.join(tmpdir, base_name)
+            with zipfile.ZipFile(archive_name, "w") as current_zip:
+                zip_content = {}
+                for k, path, name in files:
+                    try:
+                        current_zip.write(path, arcname=name)
+                        zip_content[k] = name
+                    except OSError:
+                        pass
+                content_name = os.path.join(tmpdir, "content.json")
+                with open(content_name, "w") as content:
+                    content.write(json.dumps(zip_content))
+                current_zip.write(content_name, arcname="content.json")
+
+            today = datetime.date.today()
+            with open(
+                    archive_name,
+                    "rb",
+            ) as raw_file:
+                self.archive_file.save(
+                    "upload/imports/{}/{:02d}/{}".format(
+                        today.year, today.month, base_name
+                    ),
+                    File(raw_file),
+                )
+        IshtarSiteProfile = apps.get_model("ishtar_common", "IshtarSiteProfile")
+        profile = IshtarSiteProfile.get_current_profile()
+        if profile.delete_image_zip_on_archive:
+            sub_import_file_attr.append("imported_images")
+        for attr in file_attr:
+            file_field = getattr(self, attr)
+            if file_field:
+                try:
+                    os.remove(file_field.path)
+                except FileNotFoundError:
+                    pass
+                setattr(self, attr, None)
+        sub_import_file_attr.append("imported_file")
+        if profile.delete_image_zip_on_archive:
+            sub_import_file_attr.append("imported_images")
+        for sub_import in import_list:
+            for attr in sub_import_file_attr:
+                file_field = getattr(sub_import, attr)
+                if file_field:
+                    try:
+                        os.remove(file_field.path)
+                    except FileNotFoundError:
+                        pass
+                    setattr(sub_import, attr, None)
+        self.save()
+        self._archive_pending = False
+
     def get_all_imported(self):
         imported = []
         for imp in self.imports.all():
@@ -1594,14 +1734,6 @@ class Import(BaseImport):
         max_length=255,
         help_text=max_size_help(),
     )
-    archive_file = models.FileField(
-        _("Archive file"),
-        upload_to="upload/imports/%Y/%m/",
-        blank=True,
-        null=True,
-        max_length=255,
-        help_text=max_size_help(),
-    )
     state = models.CharField(
         _("State"), max_length=2, choices=IMPORT_STATE, default="C"
     )
@@ -2313,16 +2445,6 @@ class Import(BaseImport):
         self.save()
         self._archive_pending = False
 
-    def archive(self):
-        self.state = "AC"
-        self.end_date = datetime.datetime.now()
-        self._archive()
-
-    def unarchive(self, state):
-        if not self._unarchive():
-            self.state = state
-            self.save()  # only save if no save previously
-
     def get_all_imported(self):
         imported = []
         for related, zorg in get_all_related_m2m_objects_with_model(self):
@@ -2330,15 +2452,6 @@ class Import(BaseImport):
             imported += [(accessor, obj) for obj in getattr(self, accessor).all()]
         return imported
 
-    def save(self, *args, **kwargs):
-        super(Import, self).save(*args, **kwargs)
-        if (
-            self.state == "AC"
-            and not getattr(self, "_archive_pending", False)
-            and not self.archive_file
-        ):
-            self._archive()
-
 
 def pre_delete_import(sender, **kwargs):
     # deleted imported items when an import is delete