diff options
Diffstat (limited to 'ishtar_common/serializers.py')
| -rw-r--r-- | ishtar_common/serializers.py | 130 |
1 files changed, 106 insertions, 24 deletions
diff --git a/ishtar_common/serializers.py b/ishtar_common/serializers.py index 187686321..e9b904d6f 100644 --- a/ishtar_common/serializers.py +++ b/ishtar_common/serializers.py @@ -1,3 +1,4 @@ +from collections import OrderedDict from copy import deepcopy import datetime import json @@ -9,6 +10,7 @@ from rest_framework import serializers from zipfile import ZipFile from django.apps import apps +from django.conf import settings from django.contrib.sites.models import Site from django.core.serializers import deserialize, serialize @@ -72,11 +74,12 @@ def archive_serialization(result, archive_dir=None, archive=False, ) if not archive_name.endswith(".zip"): archive_name += ".zip" + mode = "w" if archive_created else "a" with tempfile.TemporaryDirectory() as tmpdirname: if archive_dir: os.mkdir(tmpdirname + os.sep + archive_dir) - with ZipFile(archive_name, 'w') as current_zip: + with ZipFile(archive_name, mode) as current_zip: if archive_created: base_filename = "info.json" filename = tmpdirname + os.sep + base_filename @@ -86,19 +89,21 @@ def archive_serialization(result, archive_dir=None, archive=False, ) current_zip.write(filename, arcname=base_filename) - for model_name in result: + for dir_name, model_name in result: base_filename = model_name + ".json" filename = tmpdirname + os.sep + base_filename with open(filename, "w") as json_file: - json_file.write(result[model_name]) - current_zip.write(filename, - arcname="types" + os.sep + base_filename) + json_file.write(result[(dir_name, model_name)]) + arcname = base_filename + if dir_name: + arcname = dir_name + os.sep + base_filename + current_zip.write(filename, arcname=arcname) return archive_name def type_serialization(archive=False, return_empty_types=False, archive_name=None): - result = {} + result = OrderedDict() for model in apps.get_models(): if not isinstance(model(), models.GeneralType): continue @@ -113,9 +118,9 @@ def type_serialization(archive=False, return_empty_types=False, recursion = "inverse_relation" if recursion: q = q.filter(**{recursion + "__isnull": True}) - result[model_name] = serialize( - "json", q.all(), - indent=2, + key = ("types", model_name) + result[key] = serialize( + "json", q.all(), indent=2, use_natural_foreign_keys=True, use_natural_primary_keys=True ) if recursion: @@ -126,9 +131,9 @@ def type_serialization(archive=False, return_empty_types=False, v = serialize( "json", q.all(), indent=2, use_natural_foreign_keys=True, use_natural_primary_keys=True) - new_result = json.loads(result[model_name]) + new_result = json.loads(result[key]) new_result += json.loads(v) - result[model_name] = json.dumps(new_result, indent=2) + result[key] = json.dumps(new_result, indent=2) serialized += [item["id"] for item in q.values("id").all()] q = base_q.filter(**{recursion + "_id__in": serialized} ).exclude(id__in=serialized) @@ -142,15 +147,78 @@ def type_serialization(archive=False, return_empty_types=False, result_cleaned = deepcopy(result_to_add) for res in result_cleaned: # first add with no recursion res["fields"][recursion] = None - new_result = json.loads(result[model_name]) + new_result = json.loads(result[key]) new_result += result_cleaned new_result += result_to_add - result[model_name] = json.dumps(new_result, indent=2) + result[key] = json.dumps(new_result, indent=2) return archive_serialization(result, archive_dir="types", archive=archive, return_empty_types=return_empty_types, archive_name=archive_name) +def generic_get_results(model_list, dirname): + result = OrderedDict() + for model in model_list: + model_name = model.__name__ + model_name = str(model.__module__).split(".")[0] + "__" + model_name + key = (dirname, model_name) + result[key] = serialize( + "json", model.objects.all(), + indent=2, + use_natural_foreign_keys=True, use_natural_primary_keys=True, + ) + if hasattr(model, "SERIALIZATION_EXCLUDE"): + new_result = json.loads(result[key]) + for idx in range(len(new_result)): + for excluded_field in model.SERIALIZATION_EXCLUDE: + new_result[idx]["fields"].pop(excluded_field) + result[key] = json.dumps(new_result) + return result + + +def generic_archive_files(model_list, archive_name=None): + result = [] + for model in model_list: + if hasattr(model, "SERIALIZATION_FILES"): + for item in model.objects.all(): + for attr in model.SERIALIZATION_FILES: + media = getattr(item, attr) + result.append((media.path, media.name)) + + archive_created = False + if not archive_name: + archive_created = True + tmpdir = tempfile.mkdtemp(prefix="ishtarexport-") + os.sep + archive_name = tmpdir + "media.zip" + if not archive_name.endswith(".zip"): + archive_name += ".zip" + mode = "w" if archive_created else "a" + with ZipFile(archive_name, mode) as current_zip: + for media_path, name in result: + current_zip.write(media_path, arcname=name) + return archive_name + + +CONF_MODEL_LIST = [ + models.IshtarSiteProfile, models.GlobalVar, models.CustomForm, + models.ExcludedField, models.JsonDataSection, models.JsonDataField, + models.CustomFormJsonField, models.ImporterModel, + models.DocumentTemplate +] + + +def conf_serialization(archive=False, return_empty_types=False, + archive_name=None): + media_archive = generic_archive_files(CONF_MODEL_LIST) + result = generic_get_results(CONF_MODEL_LIST, "common_conf") + full_archive = archive_serialization( + result, archive_dir="common_conf", archive=archive, + return_empty_types=return_empty_types, archive_name=archive_name) + with ZipFile(full_archive, 'a') as current_zip: + current_zip.write(media_archive, arcname="media.zip") + return full_archive + + def restore_serialized(archive_name, delete_existing=False): with zipfile.ZipFile(archive_name, "r") as zip_file: # check version @@ -161,14 +229,28 @@ def restore_serialized(archive_name, delete_existing=False): "installation".format(info["serialize-version"]) ) - # restore types - for json_filename in zip_file.namelist(): - path = json_filename.split(os.sep) - if len(path) != 2 or path[0] != "types": - continue - if delete_existing: - model = get_model_from_filename(path[-1]) - model.objects.all().delete() - data = zip_file.read(json_filename).decode("utf-8") - for obj in deserialize("json", data): - obj.save() + DIRS = ( + ("types", [None]), ("common_conf", CONF_MODEL_LIST) + ) + namelist = zip_file.namelist() + for current_dir, model_list in DIRS: + for current_model in model_list: + for json_filename in namelist: + path = json_filename.split(os.sep) + if len(path) != 2 or path[0] != current_dir: + continue + model = get_model_from_filename(path[-1]) + if current_model and current_model != model: + continue + if delete_existing: + model.objects.all().delete() + data = zip_file.read(json_filename).decode("utf-8") + for obj in deserialize("json", data): + obj.save() + # restore media + if "media.zip" in namelist: + with tempfile.TemporaryDirectory() as tmp_dir_name: + zip_file.extract("media.zip", tmp_dir_name) + with zipfile.ZipFile( + tmp_dir_name + os.sep + "media.zip", 'r') as media_zip: + media_zip.extractall(settings.MEDIA_ROOT)
\ No newline at end of file |