From f9dc3d253af02bdd78ad0a845d7734ec1863a9e5 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Wed, 20 Sep 2023 16:44:55 +0200 Subject: ✨ Import group: archive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ishtar_common/models_imports.py | 167 +++++++++++++++++++++++++++++++++------- 1 file changed, 140 insertions(+), 27 deletions(-) (limited to 'ishtar_common/models_imports.py') diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py index 9e505392d..b03b42e1a 100644 --- a/ishtar_common/models_imports.py +++ b/ishtar_common/models_imports.py @@ -1359,6 +1359,14 @@ class BaseImport(models.Model): max_length=220, help_text=max_size_help(), ) + archive_file = models.FileField( + _("Archive file"), + upload_to="upload/imports/%Y/%m/", + blank=True, + null=True, + max_length=255, + help_text=max_size_help(), + ) encoding = models.CharField( _("Encoding"), choices=ENCODINGS, default="utf-8", max_length=15, help_text=_("Only required for CSV file"), @@ -1384,6 +1392,7 @@ class BaseImport(models.Model): end_date = models.DateTimeField( _("End date"), auto_now_add=True, blank=True, null=True, editable=False ) + state = None class Meta: abstract = True @@ -1396,6 +1405,31 @@ class BaseImport(models.Model): def pre_import_form_is_valid(self) -> bool: raise NotImplemented() + def _archive(self): + raise NotImplemented() + + def _unarchive(self): + raise NotImplemented() + + def archive(self): + self.state = "AC" + self.end_date = datetime.datetime.now() + self._archive() + + def unarchive(self, state): + if not self._unarchive(): + self.state = state + self.save() # only save if no save previously + + def save(self, *args, **kwargs): + super().save(*args, **kwargs) + if ( + self.state == "AC" + and not getattr(self, "_archive_pending", False) + and not self.archive_file + ): + self._archive() + class ImportGroup(BaseImport): importer_type = models.ForeignKey(ImporterGroup, on_delete=models.CASCADE, @@ -1508,6 +1542,112 @@ class ImportGroup(BaseImport): self.end_date = datetime.datetime.now() self.save() + def _unarchive(self): + if not self.archive_file: + return + with tempfile.TemporaryDirectory() as tmp_dir_name: + # extract the current archive + current_zip = zipfile.ZipFile(self.archive_file.path, "r") + name_list = current_zip.namelist() + if "content.json" not in name_list: + return + for name in name_list: + current_zip.extract(name, tmp_dir_name) + current_zip.close() + content_name = os.path.join(tmp_dir_name, "content.json") + try: + with open(content_name, "r") as content: + files = json.loads(content.read()) + except (IOError, json.JSONDecodeError): + return + today = datetime.date.today() + for attr in files: + filename = files[attr] + full_filename = os.path.join(tmp_dir_name, filename) + with open(full_filename, "rb") as raw_file: + getattr(self, attr).save( + "upload/imports/{}/{:02d}/{}".format( + today.year, today.month, filename + ), + File(raw_file), + ) + + os.remove(self.archive_file.path) + setattr(self, "archive_file", None) + self.state = "FE" if self.error_file else "F" + self.save() + return True + + def _archive(self): + file_attr = ["imported_file"] + sub_import_file_attr = ["error_file", "result_file", "match_file"] + files = [ + (k, getattr(self, k).path, getattr(self, k).name.split(os.sep)[-1]) + for k in file_attr + if getattr(self, k) + ] + import_list = self.import_list() + for idx, sub_import in enumerate(import_list): + files += [ + (f"sub-{idx}-{k}", getattr(sub_import, k).path, + getattr(sub_import, k).name.split(os.sep)[-1]) + for k in sub_import_file_attr + if getattr(sub_import, k) + ] + with tempfile.TemporaryDirectory("-ishtar") as tmpdir: + base_name = "{}.zip".format(slugify(self.name)) + archive_name = os.path.join(tmpdir, base_name) + with zipfile.ZipFile(archive_name, "w") as current_zip: + zip_content = {} + for k, path, name in files: + try: + current_zip.write(path, arcname=name) + zip_content[k] = name + except OSError: + pass + content_name = os.path.join(tmpdir, "content.json") + with open(content_name, "w") as content: + content.write(json.dumps(zip_content)) + current_zip.write(content_name, arcname="content.json") + + today = datetime.date.today() + with open( + archive_name, + "rb", + ) as raw_file: + self.archive_file.save( + "upload/imports/{}/{:02d}/{}".format( + today.year, today.month, base_name + ), + File(raw_file), + ) + IshtarSiteProfile = apps.get_model("ishtar_common", "IshtarSiteProfile") + profile = IshtarSiteProfile.get_current_profile() + if profile.delete_image_zip_on_archive: + sub_import_file_attr.append("imported_images") + for attr in file_attr: + file_field = getattr(self, attr) + if file_field: + try: + os.remove(file_field.path) + except FileNotFoundError: + pass + setattr(self, attr, None) + sub_import_file_attr.append("imported_file") + if profile.delete_image_zip_on_archive: + sub_import_file_attr.append("imported_images") + for sub_import in import_list: + for attr in sub_import_file_attr: + file_field = getattr(sub_import, attr) + if file_field: + try: + os.remove(file_field.path) + except FileNotFoundError: + pass + setattr(sub_import, attr, None) + self.save() + self._archive_pending = False + def get_all_imported(self): imported = [] for imp in self.imports.all(): @@ -1594,14 +1734,6 @@ class Import(BaseImport): max_length=255, help_text=max_size_help(), ) - archive_file = models.FileField( - _("Archive file"), - upload_to="upload/imports/%Y/%m/", - blank=True, - null=True, - max_length=255, - help_text=max_size_help(), - ) state = models.CharField( _("State"), max_length=2, choices=IMPORT_STATE, default="C" ) @@ -2313,16 +2445,6 @@ class Import(BaseImport): self.save() self._archive_pending = False - def archive(self): - self.state = "AC" - self.end_date = datetime.datetime.now() - self._archive() - - def unarchive(self, state): - if not self._unarchive(): - self.state = state - self.save() # only save if no save previously - def get_all_imported(self): imported = [] for related, zorg in get_all_related_m2m_objects_with_model(self): @@ -2330,15 +2452,6 @@ class Import(BaseImport): imported += [(accessor, obj) for obj in getattr(self, accessor).all()] return imported - def save(self, *args, **kwargs): - super(Import, self).save(*args, **kwargs) - if ( - self.state == "AC" - and not getattr(self, "_archive_pending", False) - and not self.archive_file - ): - self._archive() - def pre_delete_import(sender, **kwargs): # deleted imported items when an import is delete -- cgit v1.2.3