diff options
Diffstat (limited to 'ishtar_common/models_imports.py')
| -rw-r--r-- | ishtar_common/models_imports.py | 107 |
1 files changed, 76 insertions, 31 deletions
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py index b03b42e1a..78398ea1a 100644 --- a/ishtar_common/models_imports.py +++ b/ishtar_common/models_imports.py @@ -19,7 +19,7 @@ import csv import datetime -import sys +from pathlib import Path import fiona from fiona import crs as fiona_crs @@ -1452,6 +1452,10 @@ class ImportGroup(BaseImport): return False @property + def has_error(self) -> bool: + return any(1 for imprt in self.imports.all() if imprt.has_error) + + @property def pre_import_form_is_valid(self) -> bool: return not any(-1 for imp in self.imports.all() if not imp.pre_import_form_is_valid) @@ -1545,7 +1549,9 @@ class ImportGroup(BaseImport): def _unarchive(self): if not self.archive_file: return - with tempfile.TemporaryDirectory() as tmp_dir_name: + today = datetime.date.today() + sub_imports_ids = [(Import, imp.pk) for imp in self.import_list()] + with (tempfile.TemporaryDirectory("-ishtar") as tmp_dir_name): # extract the current archive current_zip = zipfile.ZipFile(self.archive_file.path, "r") name_list = current_zip.namelist() @@ -1560,22 +1566,71 @@ class ImportGroup(BaseImport): files = json.loads(content.read()) except (IOError, json.JSONDecodeError): return - today = datetime.date.today() for attr in files: filename = files[attr] full_filename = os.path.join(tmp_dir_name, filename) - with open(full_filename, "rb") as raw_file: - getattr(self, attr).save( + current_imports_ids = [(ImportGroup, self.pk)] + + # identify imports to attach the file + if attr == "imported_file": + current_imports_ids += sub_imports_ids # imported_file is attached to each imports + elif attr.startswith("sub-"): + # identify the sub import to attach the file + # archive filename is "sub-{subimport_order}-{attribute}" + split_attr = attr.split("-") + if len(split_attr) < 3: + continue + try: + idx = int(split_attr[1]) + except (ValueError, IndexError): + continue + if idx > (len(sub_imports_ids) - 1): + continue + current_imports_ids = [sub_imports_ids[idx]] + attr = "-".join(split_attr[2:]) + + for idx, current_import_id in enumerate(current_imports_ids): + model, current_import_id = current_import_id + current_import = model.objects.get(pk=current_import_id) + path = Path(full_filename) + c_filename = path.name + if idx: + c_filename = f"{idx:02d}-" + c_filename + path = Path(full_filename) + """ + # TODO: should be enough... + # but need to be set explicitly, why? + with path.open(mode="rb") as raw_file: + getattr(current_import, attr).save(c_filename, File(raw_file)) + """ + + w_filename = os.path.join( + settings.MEDIA_ROOT, "upload/imports/{}/{:02d}/{}".format( - today.year, today.month, filename - ), - File(raw_file), + today.year, today.month, c_filename + ) ) - + with path.open(mode="rb") as raw_file: + with open(w_filename, "wb") as w_file: + w_file.write(raw_file.read()) + with open(w_filename, mode="rb") as raw_file: + f = File(raw_file, name=c_filename) + f.path = w_filename + setattr(current_import, attr, f) + current_import.save() + getattr(current_import, attr).name = w_filename + current_import.save() os.remove(self.archive_file.path) + self.refresh_from_db() setattr(self, "archive_file", None) - self.state = "FE" if self.error_file else "F" + self.state = "FE" if self.has_error else "F" self.save() + + for model, sub_import_id in sub_imports_ids: + sub_import = model.objects.get(pk=sub_import_id) + sub_import.state = "FE" if sub_import.has_error else "F" + sub_import.save() + return True def _archive(self): @@ -1610,17 +1665,11 @@ class ImportGroup(BaseImport): content.write(json.dumps(zip_content)) current_zip.write(content_name, arcname="content.json") - today = datetime.date.today() with open( archive_name, "rb", ) as raw_file: - self.archive_file.save( - "upload/imports/{}/{:02d}/{}".format( - today.year, today.month, base_name - ), - File(raw_file), - ) + self.archive_file.save(base_name, File(raw_file)) IshtarSiteProfile = apps.get_model("ishtar_common", "IshtarSiteProfile") profile = IshtarSiteProfile.get_current_profile() if profile.delete_image_zip_on_archive: @@ -1648,6 +1697,10 @@ class ImportGroup(BaseImport): self.save() self._archive_pending = False + def archive(self): + super().archive() + self.imports.update(state="AC") + def get_all_imported(self): imported = [] for imp in self.imports.all(): @@ -1820,6 +1873,10 @@ class Import(BaseImport): return bool(self.importer_type.columns.filter(col_number__lte=0).count()) @property + def has_error(self) -> bool: + return bool(self.error_file) + + @property def pre_import_form_is_valid(self) -> bool: for column in self.importer_type.columns.filter(col_number__lte=0, required=True): q = ImportColumnValue.objects.filter(column=column, import_item=self) @@ -2377,17 +2434,11 @@ class Import(BaseImport): files = json.loads(content.read()) except (IOError, json.JSONDecodeError): return - today = datetime.date.today() for attr in files: filename = files[attr] full_filename = os.path.join(tmp_dir_name, filename) with open(full_filename, "rb") as raw_file: - getattr(self, attr).save( - "upload/imports/{}/{:02d}/{}".format( - today.year, today.month, filename - ), - File(raw_file), - ) + getattr(self, attr).save(filename, File(raw_file)) os.remove(self.archive_file.path) setattr(self, "archive_file", None) @@ -2419,17 +2470,11 @@ class Import(BaseImport): content.write(json.dumps(zip_content)) current_zip.write(content_name, arcname="content.json") - today = datetime.date.today() with open( archive_name, "rb", ) as raw_file: - self.archive_file.save( - "upload/imports/{}/{:02d}/{}".format( - today.year, today.month, base_name - ), - File(raw_file), - ) + self.archive_file.save(base_name, File(raw_file)) IshtarSiteProfile = apps.get_model("ishtar_common", "IshtarSiteProfile") profile = IshtarSiteProfile.get_current_profile() if profile.delete_image_zip_on_archive: |