Serialization: manage medias, manage model order (serialization and restore), serialize conf

1 files changed, 106 insertions, 24 deletions

diff --git a/ishtar_common/serializers.py b/ishtar_common/serializers.py
index 187686321..e9b904d6f 100644
--- a/ishtar_common/serializers.py
+++ b/ishtar_common/serializers.py
@@ -1,3 +1,4 @@
+from collections import OrderedDict
 from copy import deepcopy
 import datetime
 import json
@@ -9,6 +10,7 @@ from rest_framework import serializers
 from zipfile import ZipFile
 
 from django.apps import apps
+from django.conf import settings
 from django.contrib.sites.models import Site
 from django.core.serializers import deserialize, serialize
 
@@ -72,11 +74,12 @@ def archive_serialization(result, archive_dir=None, archive=False,
         )
     if not archive_name.endswith(".zip"):
         archive_name += ".zip"
+    mode = "w" if archive_created else "a"
     with tempfile.TemporaryDirectory() as tmpdirname:
         if archive_dir:
             os.mkdir(tmpdirname + os.sep + archive_dir)
 
-        with ZipFile(archive_name, 'w') as current_zip:
+        with ZipFile(archive_name, mode) as current_zip:
             if archive_created:
                 base_filename = "info.json"
                 filename = tmpdirname + os.sep + base_filename
@@ -86,19 +89,21 @@ def archive_serialization(result, archive_dir=None, archive=False,
                     )
                 current_zip.write(filename, arcname=base_filename)
 
-            for model_name in result:
+            for dir_name, model_name in result:
                 base_filename = model_name + ".json"
                 filename = tmpdirname + os.sep + base_filename
                 with open(filename, "w") as json_file:
-                    json_file.write(result[model_name])
-                current_zip.write(filename,
-                                  arcname="types" + os.sep + base_filename)
+                    json_file.write(result[(dir_name, model_name)])
+                arcname = base_filename
+                if dir_name:
+                    arcname = dir_name + os.sep + base_filename
+                current_zip.write(filename, arcname=arcname)
     return archive_name
 
 
 def type_serialization(archive=False, return_empty_types=False,
                        archive_name=None):
-    result = {}
+    result = OrderedDict()
     for model in apps.get_models():
         if not isinstance(model(), models.GeneralType):
             continue
@@ -113,9 +118,9 @@ def type_serialization(archive=False, return_empty_types=False,
             recursion = "inverse_relation"
         if recursion:
             q = q.filter(**{recursion + "__isnull": True})
-        result[model_name] = serialize(
-            "json", q.all(),
-            indent=2,
+        key = ("types", model_name)
+        result[key] = serialize(
+            "json", q.all(), indent=2,
             use_natural_foreign_keys=True, use_natural_primary_keys=True
         )
         if recursion:
@@ -126,9 +131,9 @@ def type_serialization(archive=False, return_empty_types=False,
                 v = serialize(
                     "json", q.all(), indent=2, use_natural_foreign_keys=True,
                     use_natural_primary_keys=True)
-                new_result = json.loads(result[model_name])
+                new_result = json.loads(result[key])
                 new_result += json.loads(v)
-                result[model_name] = json.dumps(new_result, indent=2)
+                result[key] = json.dumps(new_result, indent=2)
                 serialized += [item["id"] for item in q.values("id").all()]
                 q = base_q.filter(**{recursion + "_id__in": serialized}
                                   ).exclude(id__in=serialized)
@@ -142,15 +147,78 @@ def type_serialization(archive=False, return_empty_types=False,
                 result_cleaned = deepcopy(result_to_add)
                 for res in result_cleaned:  # first add with no recursion
                     res["fields"][recursion] = None
-                new_result = json.loads(result[model_name])
+                new_result = json.loads(result[key])
                 new_result += result_cleaned
                 new_result += result_to_add
-                result[model_name] = json.dumps(new_result, indent=2)
+                result[key] = json.dumps(new_result, indent=2)
     return archive_serialization(result, archive_dir="types", archive=archive,
                                  return_empty_types=return_empty_types,
                                  archive_name=archive_name)
 
 
+def generic_get_results(model_list, dirname):
+    result = OrderedDict()
+    for model in model_list:
+        model_name = model.__name__
+        model_name = str(model.__module__).split(".")[0] + "__" + model_name
+        key = (dirname, model_name)
+        result[key] = serialize(
+            "json", model.objects.all(),
+            indent=2,
+            use_natural_foreign_keys=True, use_natural_primary_keys=True,
+        )
+        if hasattr(model, "SERIALIZATION_EXCLUDE"):
+            new_result = json.loads(result[key])
+            for idx in range(len(new_result)):
+                for excluded_field in model.SERIALIZATION_EXCLUDE:
+                    new_result[idx]["fields"].pop(excluded_field)
+            result[key] = json.dumps(new_result)
+    return result
+
+
+def generic_archive_files(model_list, archive_name=None):
+    result = []
+    for model in model_list:
+        if hasattr(model, "SERIALIZATION_FILES"):
+            for item in model.objects.all():
+                for attr in model.SERIALIZATION_FILES:
+                    media = getattr(item, attr)
+                    result.append((media.path, media.name))
+
+    archive_created = False
+    if not archive_name:
+        archive_created = True
+        tmpdir = tempfile.mkdtemp(prefix="ishtarexport-") + os.sep
+        archive_name = tmpdir + "media.zip"
+    if not archive_name.endswith(".zip"):
+        archive_name += ".zip"
+    mode = "w" if archive_created else "a"
+    with ZipFile(archive_name, mode) as current_zip:
+        for media_path, name in result:
+            current_zip.write(media_path, arcname=name)
+    return archive_name
+
+
+CONF_MODEL_LIST = [
+    models.IshtarSiteProfile, models.GlobalVar, models.CustomForm,
+    models.ExcludedField, models.JsonDataSection, models.JsonDataField,
+    models.CustomFormJsonField, models.ImporterModel,
+    models.DocumentTemplate
+]
+
+
+def conf_serialization(archive=False, return_empty_types=False,
+                       archive_name=None):
+    media_archive = generic_archive_files(CONF_MODEL_LIST)
+    result = generic_get_results(CONF_MODEL_LIST, "common_conf")
+    full_archive = archive_serialization(
+        result, archive_dir="common_conf", archive=archive,
+        return_empty_types=return_empty_types, archive_name=archive_name)
+    with ZipFile(full_archive, 'a') as current_zip:
+        current_zip.write(media_archive, arcname="media.zip")
+    return full_archive
+
+
 def restore_serialized(archive_name, delete_existing=False):
     with zipfile.ZipFile(archive_name, "r") as zip_file:
         # check version
@@ -161,14 +229,28 @@ def restore_serialized(archive_name, delete_existing=False):
                 "installation".format(info["serialize-version"])
             )
 
-        # restore types
-        for json_filename in zip_file.namelist():
-            path = json_filename.split(os.sep)
-            if len(path) != 2 or path[0] != "types":
-                continue
-            if delete_existing:
-                model = get_model_from_filename(path[-1])
-                model.objects.all().delete()
-            data = zip_file.read(json_filename).decode("utf-8")
-            for obj in deserialize("json", data):
-                obj.save()
+        DIRS = (
+            ("types", [None]), ("common_conf", CONF_MODEL_LIST)
+        )
+        namelist = zip_file.namelist()
+        for current_dir, model_list in DIRS:
+            for current_model in model_list:
+                for json_filename in namelist:
+                    path = json_filename.split(os.sep)
+                    if len(path) != 2 or path[0] != current_dir:
+                        continue
+                    model = get_model_from_filename(path[-1])
+                    if current_model and current_model != model:
+                        continue
+                    if delete_existing:
+                        model.objects.all().delete()
+                    data = zip_file.read(json_filename).decode("utf-8")
+                    for obj in deserialize("json", data):
+                        obj.save()
+        # restore media
+        if "media.zip" in namelist:
+            with tempfile.TemporaryDirectory() as tmp_dir_name:
+                zip_file.extract("media.zip", tmp_dir_name)
+                with zipfile.ZipFile(
+                        tmp_dir_name + os.sep + "media.zip", 'r') as media_zip:
+                    media_zip.extractall(settings.MEDIA_ROOT)
\ No newline at end of file