path: root/ishtar_common/wizards.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2010-2017  Étienne Loks  <etienne.loks_AT_peacefrogsDOTnet>

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# See the file COPYING for details.

import datetime
import logging
import os

# from functools import wraps

from django.conf import settings
from django.contrib import messages
from formtools.wizard.views import (
    NamedUrlWizardView,
    normalize_name,
    get_storage,
    StepsHelper,
)

from django.contrib.sites.models import Site
from django.core.exceptions import ObjectDoesNotExist
from django.core.files.images import ImageFile
from django.core.files.storage import default_storage, FileSystemStorage
from django.core.mail import send_mail
from django.db.models.fields.files import FileField, ImageFieldFile
from django.db.models.fields.related import ManyToManyField
from django.db.models.fields import NOT_PROVIDED

from django.http import HttpResponseRedirect, Http404
from django.forms import ValidationError
from django.shortcuts import redirect, render, reverse
from django.template import loader
from django.utils.translation import ugettext_lazy as _
from django.utils.safestring import mark_safe

from ishtar_common import models
from ishtar_common.forms import CustomForm, reverse_lazy
from ishtar_common.utils import get_all_field_names, MultiValueDict, put_session_message

logger = logging.getLogger(__name__)

# buggy and unecessary at least for the moment...
"""
def _check_right(step, condition=True):
    '''Return a method to check the right for a specific step'''
    def check_right(self):
        cond = condition
        if callable(condition):
            cond = condition(self)
        if not cond:
            return False
        return True
        # TODO: to be check
        if not hasattr(self.request.user, 'ishtaruser'):
            return False
        return self.request.user.ishtaruser.has_right(
            ('administrator', step), session=self.request.session)
    return check_right
"""


def filter_no_fields_form(form, other_check=None):
    def func(self):
        if not hasattr(self.request.user, "ishtaruser"):
            return False
        if issubclass(form, CustomForm):
            enabled, excluded, json_fields = form.check_custom_form(
                self.request.user.ishtaruser
            )
            if not hasattr(self, "json_fields"):
                self.json_fields = {}
            self.json_fields[form.form_slug] = [
                key for order, key, field in json_fields
            ]
            if not enabled:
                return False
        if other_check:
            return other_check(self)
        return True

    return func


EXTRA_FORM_MODALS = ["container", "warehouse", "person", "organization"]


class IshtarWizard(NamedUrlWizardView):
    def get_form_kwargs(self, step=None):
        kwargs = super(IshtarWizard, self).get_form_kwargs(step)
        if (
            hasattr(self.form_list[step], "need_user_for_initialization")
            and self.form_list[step].need_user_for_initialization
        ):
            kwargs["user"] = self.request.user
        return kwargs

    def get_context_data(self, form, **kwargs):
        context = super(IshtarWizard, self).get_context_data(form, **kwargs)

        context["extra_form_modals"] = (
            form.extra_form_modals
            if hasattr(form, "extra_form_modals")
            else EXTRA_FORM_MODALS
        )

        open_item_id = self.request.GET.get("open_item", None)
        if open_item_id and self.model and getattr(self.model, "SHOW_URL", None):
            url = reverse(self.model.SHOW_URL, args=[open_item_id])
            if not url.endswith("/"):
                url += "/"
            context["open_url"] = url

        return context


class Wizard(IshtarWizard):
    model = None
    label = ""
    translated_keys = []
    modification = None  # True when the wizard modify an item
    storage_name = "formtools.wizard.storage.session.SessionStorage"
    wizard_done_template = "ishtar/wizard/wizard_done.html"
    wizard_done_window = ""
    redirect_url = None
    open_created_in_redirect = True
    wizard_confirm = "ishtar/wizard/confirm_wizard.html"
    wizard_templates = {}
    filter_owns = {}
    current_obj_slug = ""
    current_object_key = "pk"
    ignore_init_steps = []
    file_storage = FileSystemStorage(location=os.path.join(settings.MEDIA_ROOT, "tmp"))
    main_item_select_keys = ("selec-",)
    formset_pop_deleted = True
    alt_is_own_method = None  # alternate method name for "is_own" check

    saved_args = {}  # argument to pass on object save

    def get_label(self):
        return self.label

    '''
    # buggy and unecessary...
    def __init__(self, *args, **kwargs):
        """Check right for each step of the wizard"""
        super(Wizard, self).__init__(*args, **kwargs)
        for form_key in self.form_list.keys()[:-1]:
            condition = self.condition_dict.get(form_key, True)
            cond = _check_right(form_key, condition)
            self.condition_dict[form_key] = cond
    '''

    @classmethod
    def get_initkwargs(cls, *args, **kwargs):
        kwargs = super(Wizard, cls).get_initkwargs(*args, **kwargs)
        # remove
        for form_key in kwargs["form_list"]:
            form = kwargs["form_list"][form_key]
            other_check = None
            if form_key in kwargs["condition_dict"]:
                other_check = kwargs["condition_dict"][form_key]
            kwargs["condition_dict"][form_key] = filter_no_fields_form(
                form, other_check
            )
        return kwargs

    def check_own_permissions(self, request, step=None, *args, **kwargs):
        # reinit default dispatch of a wizard - not clean...
        self.request = request
        self.session = request.session
        self.prefix = self.get_prefix(request, *args, **kwargs)
        self.storage = get_storage(
            self.storage_name, self.prefix, request, getattr(self, "file_storage", None)
        )
        self.steps = StepsHelper(self)

        current_object = self.get_current_object()
        ishtaruser = (
            request.user.ishtaruser if hasattr(request.user, "ishtaruser") else None
        )

        # not the first step and current object is not owned
        if self.steps and self.steps.first != step and current_object:
            is_own = current_object.is_own(
                ishtaruser, alt_query_own=self.alt_is_own_method
            )
            if not is_own:
                messages.add_message(
                    request,
                    messages.WARNING,
                    _("Permission error: you cannot do this action."),
                )
                self.session_reset(request, self.url_name)
                return
        return True

    def dispatch(self, request, *args, **kwargs):
        self.current_right = kwargs.get("current_right", None)
        step = kwargs.get("step", None)
        # check that the current object is really owned by the current user
        if step and self.current_right and "_own_" in self.current_right:
            if not self.check_own_permissions(request, *args, **kwargs):
                return HttpResponseRedirect("/")
            # extra filter on forms
            self.filter_owns_items = True
        else:
            self.filter_owns_items = False

        return super(Wizard, self).dispatch(request, *args, **kwargs)

    def get_prefix(self, request, *args, **kwargs):
        """As the class name can interfere when reused prefix with the url_name"""
        return self.url_name + super(Wizard, self).get_prefix(request, *args, **kwargs)

    def get_wizard_name(self):
        """As the class name can interfere when reused, use the url_name"""
        return self.url_name

    def get_template_names(self):
        templates = ["ishtar/wizard/default_wizard.html"]
        current_step = self.steps.current
        wizard_templates = dict(
            [
                (key % {"url_name": self.url_name}, self.wizard_templates[key])
                for key in self.wizard_templates
            ]
        )
        if current_step in wizard_templates:
            templates = [wizard_templates[current_step]] + templates
        elif current_step == self.steps.last:
            templates = [self.wizard_confirm] + templates
        return templates

    def get_ignore_init_steps(self):
        return ["{}-{}".format(step, self.url_name) for step in self.ignore_init_steps]

    def get_context_data(self, form, **kwargs):
        """
        Add previous, next and current steps to manage the wizard path
        """
        context = super(Wizard, self).get_context_data(form)
        self.request.session["CURRENT_ACTION"] = self.get_wizard_name()
        step = self.steps.first
        current_step = self.steps.current

        dct = {
            "current_step_label": self.form_list[current_step].form_label,
            "wizard_label": self.get_label(),
            "current_object": self.get_current_object(),
            "is_search": bool(
                [k for k in self.main_item_select_keys if current_step.startswith(k)]
            )
            if current_step
            else False,
        }

        context.update(dct)
        if step == current_step:
            return context

        previous_steps, next_steps, previous_step_counter = [], [], 0
        while step:
            if step == current_step or (
                previous_steps and previous_steps[-1] == self.form_list[step]
            ):
                break
            previous_steps.append(self.form_list[step].form_label)
            previous_step_counter += 1
            if previous_step_counter >= len(self.steps):
                break
            step = self.steps.all[previous_step_counter]

        context.update(
            {
                "previous_steps": previous_steps,
                "previous_step_counter": previous_step_counter,
            }
        )
        storage = self.storage
        # if modification: show the next steps
        # if self.modification or True:
        next_step = self.steps.first
        current_step_passed, no_next = False, False

        # force rechecking of conditions
        self.get_form_list()
        last_step_is_available = True

        while next_step:

            # check if the form is initialized otherwise initialize it
            # put initialized data in session
            if self.modification and not storage.get_step_data(next_step):
                values = self.get_form_initial(next_step)
                prefixed_values = MultiValueDict()
                if not isinstance(values, list):
                    # simple form
                    for key in values:
                        form_key = next_step + "-" + key
                        if isinstance(values, MultiValueDict):
                            prefixed_values.setlist(form_key, values.getlist(key))
                        else:
                            prefixed_values[form_key] = values[key]
                else:
                    # formset
                    for formset_idx, v in enumerate(values):
                        prefix = "-%d-" % formset_idx
                        for key in v:
                            form_key = next_step + prefix + key
                            if isinstance(v, MultiValueDict):
                                prefixed_values.setlist(form_key, v.getlist(key))
                            else:
                                prefixed_values[form_key] = v[key]

                if (
                    not prefixed_values
                    and next_step not in self.get_ignore_init_steps()
                ):
                    # simulate a non empty data for form that might be
                    # valid when empty
                    prefixed_values["__non_empty_data"] = ""
                self.prepare_serialization(prefixed_values)
                storage.set_step_data(next_step, prefixed_values)

            if step == next_step:
                current_step_passed = True
            elif current_step_passed:
                next_steps.append(self.form_list[next_step].form_label)

            # creation
            if not self.modification:
                form_obj = self.get_form(step=next_step)
                if current_step_passed:
                    initialise_data = False
                    # formsets are considered not required
                    if hasattr(form_obj, "fields"):
                        # display next step until a required field is met
                        if [
                            field_key
                            for field_key in form_obj.fields
                            if form_obj.fields[field_key].required
                        ]:
                            no_next = True
                        elif next_step not in self.get_ignore_init_steps():
                            initialise_data = True
                    else:
                        initialise_data = True
                    if initialise_data:
                        # simulate a non empty data for form that might be
                        # valid when empty
                        prefixed_values = MultiValueDict()
                        prefixed_values["__non_empty_data"] = ""
                        storage.set_step_data(next_step, prefixed_values)

            next_step = self.get_next_step(next_step)
            if no_next:
                last_step_is_available = False
                break
        context.update(
            {"next_steps": next_steps, "last_step_is_available": last_step_is_available}
        )
        # not last step: validation
        if current_step != self.steps.last:
            return context

        final_form_list = []
        for form_key in self.get_form_list().keys():
            form_obj = self.get_form(
                step=form_key,
                data=self.storage.get_step_data(form_key),
                files=self.storage.get_step_files(form_key),
            )
            form_obj.is_valid()
            final_form_list.append(form_obj)
        last_form = final_form_list[-1]
        context.update({"datas": self.get_formated_datas(final_form_list)})
        if hasattr(last_form, "confirm_msg"):
            context.update({"confirm_msg": last_form.confirm_msg})
        if hasattr(last_form, "confirm_end_msg"):
            context.update({"confirm_end_msg": last_form.confirm_end_msg})
        return context

    def get_formated_datas(self, forms):
        """Get the data to present in the last page"""
        datas = []
        for form in forms:
            is_formset = hasattr(form, "forms")
            base_form = is_formset and form.forms[0] or form
            associated_models = (
                hasattr(base_form, "associated_models")
                and base_form.associated_models
                or {}
            )
            if not hasattr(form, "cleaned_data") and hasattr(form, "forms"):
                cleaned_datas = [
                    frm.cleaned_data for frm in form.forms if frm.is_valid()
                ]
                if not cleaned_datas:
                    continue
            elif not hasattr(form, "cleaned_data"):
                continue
            else:
                cleaned_datas = (
                    type(form.cleaned_data) == list
                    and form.cleaned_data
                    or [form.cleaned_data]
                )
            if hasattr(base_form, "get_formated_datas"):
                datas.append(
                    (form.form_label, base_form.get_formated_datas(cleaned_datas))
                )
                continue
            form_datas = []
            for cleaned_data in cleaned_datas:
                if not cleaned_data:
                    continue
                current_form_data = []
                items = (
                    hasattr(base_form, "fields")
                    and base_form.fields.keys()
                    or cleaned_data.keys()
                )
                is_deleted = False
                for key in items:
                    lbl = None
                    if key.startswith("hidden_"):
                        continue
                    if hasattr(base_form, "fields") and key in base_form.fields:
                        lbl = base_form.fields[key].label
                        if (
                            hasattr(base_form, "associated_labels")
                            and key in base_form.associated_labels
                        ):
                            lbl = base_form.associated_labels[key]
                    if key not in cleaned_data:  # no value
                        continue
                    value = cleaned_data[key]
                    if key == "DELETE" and value:
                        is_deleted = True
                    # no display when no label or no value
                    if not lbl or not lbl.strip() or value is None or value == "":
                        continue
                    if key in self.translated_keys:
                        value = _(value)
                    if type(value) == bool:
                        if value:
                            value = _("Yes")
                        else:
                            value = _("No")
                    elif key in associated_models:
                        values = []
                        if type(value) in (tuple, list):
                            values = value
                        elif "," in str(value):
                            values = str(value).strip("[").strip("]").split(",")
                        else:
                            values = [value]
                        rendered_values = []
                        for val in values:
                            item = associated_models[key].objects.get(pk=val)
                            if hasattr(item, "short_label"):
                                value = str(item.short_label)
                            else:
                                value = str(item)
                            rendered_values.append(value)
                        value = " ; ".join(rendered_values)
                    current_form_data.append((lbl, value, ""))

                if is_formset:
                    # regroup each line
                    displayed_values = []
                    for lbl, value, cls in current_form_data:
                        displayed_value = ""
                        if lbl.strip():
                            displayed_value = "<strong>{}{}</strong> ".format(
                                lbl, _(":")
                            )
                        displayed_value += str(value)
                        displayed_values.append(displayed_value)
                    value = " ; ".join(displayed_values)
                    if is_deleted:
                        value = "<span class='text-danger'>{}</span>".format(value)
                    deleted = (
                        "<span class='text-danger'>{}</span>".format(_("Deleted"))
                        if is_deleted
                        else ""
                    )
                    form_datas.append((mark_safe(deleted), mark_safe(value), ""))
                else:
                    form_datas += current_form_data

            if not form_datas:
                continue
            datas.append((form.form_label, form_datas))
        return datas

    def get_extra_model(self, dct, m2m, form_list):
        dct["history_modifier"] = self.request.user
        return dct

    def done(self, form_list, return_object=False, **kwargs):
        """Save to the model"""
        dct, m2m, whole_associated_models = {}, [], []
        # base_model for M2M
        for form in form_list:
            if not form.is_valid():
                return self.render(form)
            if hasattr(form, "readonly") and form.readonly:
                continue
            base_form = hasattr(form, "forms") and form.forms[0] or form
            associated_models = (
                hasattr(base_form, "associated_models")
                and base_form.associated_models
                or {}
            )
            if hasattr(form, "forms"):
                multi = False
                if form.forms:
                    frm = form.forms[0]
                    if hasattr(frm, "base_model") and frm.base_model:
                        whole_associated_models.append(frm.base_model)
                    elif hasattr(frm, "base_models") and frm.base_models:
                        whole_associated_models += frm.base_models
                    else:
                        whole_associated_models += associated_models.keys()
                    fields = frm.fields.copy()
                    if "DELETE" in fields:
                        fields.pop("DELETE")
                    multi = len(fields) > 1
                    if multi:
                        assert hasattr(frm, "base_model") or hasattr(
                            frm, "base_models"
                        ), "Must define a base_model(s) for " + str(frm.__class__)
                for frm in form.forms:
                    if not frm.is_valid():
                        continue
                    vals = {}
                    if "DELETE" in frm.cleaned_data:
                        if frm.cleaned_data["DELETE"]:
                            continue
                        frm.cleaned_data.pop("DELETE")
                    for key in frm.cleaned_data:
                        value = frm.cleaned_data[key]
                        dct_key = key
                        value_is_empty = value is None or value in [""]
                        if value_is_empty:
                            if multi:
                                dct_key = "__empty-" + key
                            else:
                                continue
                        if not value_is_empty and key in associated_models:
                            if type(value) in (tuple, list):
                                value = [
                                    associated_models[key].objects.get(pk=v)
                                    for v in value
                                ]
                            else:
                                value = associated_models[key].objects.get(pk=value)
                        if multi:
                            vals[dct_key] = value
                        else:
                            m2m.append((key, value))
                    if multi and vals:
                        if hasattr(frm, "base_models"):
                            for m in frm.base_models:
                                m2m.append((frm.base_model, m))
                        else:
                            m2m.append((frm.base_model, vals))
            elif type(form.cleaned_data) == dict:
                for key in form.cleaned_data:
                    if key.startswith("hidden_"):
                        continue
                    value = form.cleaned_data[key]
                    if key in associated_models:
                        if value:
                            model = associated_models[key]
                            if isinstance(value, str) and "," in value:
                                value = value.split(",")
                            if isinstance(value, list) or isinstance(value, tuple):
                                value = [
                                    model.objects.get(pk=val) for val in value if val
                                ]
                                if len(value) == 1:
                                    value = value[0]
                            else:
                                value = model.objects.get(pk=value)
                        else:
                            value = None
                    if (
                        hasattr(form, "base_model")
                        and form.base_model
                        and form.base_model == key
                    ) or (hasattr(form, "base_models") and key in form.base_models):
                        whole_associated_models.append(key)
                        if value:
                            vals = value
                            if type(vals) not in (list, tuple):
                                vals = [vals]
                            for val in vals:
                                m2m.append((key, val))
                    else:
                        dct[key] = value
        return self.save_model(
            dct, m2m, whole_associated_models, form_list, return_object
        )

    def get_saved_model(self):
        "Permit a distinguo when saved model is not the base selected model"
        return self.model

    def get_current_saved_object(self):
        "Permit a distinguo when saved model is not the base selected model"
        return self.get_current_object()

    def save_model(self, dct, m2m, whole_associated_models, form_list, return_object):
        dct = self.get_extra_model(dct, m2m, form_list)
        obj = self.get_current_saved_object()
        data = {}
        if obj and hasattr(obj, "data"):
            data = obj.data

        # manage dependant items and json fields
        other_objs = {}
        for k in list(dct.keys()):
            if "__" not in k:
                continue
            # manage json field
            if k.startswith("data__"):
                data_keys = k[len("data__") :].split("__")
                # tree
                current_data = data
                for data_key in data_keys[:-1]:
                    if data_key not in current_data:
                        current_data[data_key] = {}
                    current_data = current_data[data_key]
                value = dct.pop(k)
                if isinstance(value, datetime.datetime):
                    value = value.strftime("%Y-%m-%dT%H:%M:%S")
                elif isinstance(value, datetime.date):
                    value = value.strftime("%Y-%m-%d")
                elif value is None:
                    value = ""
                current_data[data_keys[-1]] = value
                continue

            vals = k.split("__")
            assert len(vals) == 2, "Only one level of dependant item is managed"
            dependant_item, key = vals
            if dependant_item not in other_objs:
                other_objs[dependant_item] = {}
            other_objs[dependant_item][key] = dct.pop(k)
        if obj:
            for k in list(dct.keys()):
                if k.startswith("pk"):
                    continue
                if k not in get_all_field_names(obj.__class__):
                    continue
                # False set to None for images and files
                if not k.endswith("_id") and (
                    isinstance(obj.__class__._meta.get_field(k), FileField)
                    or isinstance(obj.__class__._meta.get_field(k), ImageFile)
                ):
                    if not dct[k]:
                        dct[k] = None
                if not k.endswith("_id") and (
                    isinstance(obj.__class__._meta.get_field(k), ManyToManyField)
                ):
                    if not dct[k]:
                        dct[k] = []
                    elif type(dct[k]) not in (list, tuple):
                        dct[k] = [dct[k]]
                setattr(obj, k, dct[k])
            if hasattr(obj, "data"):
                obj.data = data
            if hasattr(obj, "pre_save"):
                obj.pre_save()
            try:
                obj.full_clean()
            except ValidationError as e:
                logger.warning(str(e))
                put_session_message(self.request.session.session_key, str(e), "error")
                return self.render(list(form_list)[-1])
            for dependant_item in other_objs:
                c_item = getattr(obj, dependant_item)
                if callable(c_item):
                    c_item = c_item()
                # manage ManyToMany if only one associated
                if hasattr(c_item, "all"):
                    c_items = c_item.all()
                    if len(c_items) != 1:
                        continue
                    c_item = c_items[0]
                if c_item:
                    # to check #
                    for k in other_objs[dependant_item]:
                        setattr(c_item, k, other_objs[dependant_item][k])
                    c_item.save()
                else:
                    m = getattr(self.model, dependant_item)
                    if callable(m):
                        m = m()
                    if hasattr(m, "related"):
                        c_item = m.related.model(**other_objs[dependant_item])
                        setattr(obj, dependant_item, c_item)
                obj.save()
            obj.save()
        else:
            adds = {}
            # manage attributes relations
            if hasattr(self.model, "ATTRS_EQUIV"):
                for k in list(other_objs.keys()):
                    if k in self.model.ATTRS_EQUIV:
                        new_k = self.model.ATTRS_EQUIV[k]
                        if new_k in other_objs:
                            other_objs[new_k].update(other_objs[k])
                        else:
                            other_objs[new_k] = other_objs[k].copy()
            for dependant_item in other_objs:
                if (
                    hasattr(self.model, "ATTRS_EQUIV")
                    and dependant_item in self.model.ATTRS_EQUIV
                ):
                    continue
                m = getattr(self.model, dependant_item)
                if callable(m):
                    m = m()
                model = m.field.rel.to
                c_dct = other_objs[dependant_item].copy()
                if issubclass(model, models.BaseHistorizedItem):
                    c_dct["history_modifier"] = self.request.user
                c_item = model(**c_dct)
                c_item.save()
                if hasattr(m, "through"):
                    adds[dependant_item] = c_item
                elif hasattr(m, "field"):
                    dct[dependant_item] = c_item
            if "pk" in dct:
                dct.pop("pk")
            # remove non relevant fields
            all_field_names = get_all_field_names(self.get_saved_model())
            for k in dct.copy():
                if (
                    not (k.endswith("_id") and k[:-3] in all_field_names)
                    and k not in all_field_names
                    and (
                        not hasattr(self.get_saved_model(), "EXTRA_SAVED_KEYS")
                        or k not in self.get_saved_model().EXTRA_SAVED_KEYS
                    )
                ):
                    dct.pop(k)
            saved_args = self.saved_args.copy()
            for k in saved_args:
                if k in dct:
                    saved_args[k] = dct.pop(k)
            obj = self.get_saved_model()(**dct)
            if hasattr(obj, "pre_save"):
                obj.pre_save()
            try:
                obj.full_clean()
            except ValidationError as e:
                logger.warning(str(e))
                put_session_message(self.request.session.session_key, str(e), "error")
                return self.render(list(form_list)[-1])
            if hasattr(obj, "data"):
                obj.data = data
            obj.save(**saved_args)
            for k in adds:
                getattr(obj, k).add(adds[k])
            # necessary to manage interaction between models like
            # material_index management for baseitems
            obj._cached_label_checked = False
            obj.save()
        m2m_items = {}
        # clear
        # TODO! perf - to be really optimized
        old_m2ms = {}
        for model in whole_associated_models:
            related_model = getattr(obj, model + "s")
            # manage through
            if hasattr(related_model, "through") and related_model.through:
                if hasattr(related_model.through, "RELATED_SET_NAME"):
                    related_set_name = related_model.through.RELATED_SET_NAME
                else:
                    related_set_name = str(
                        related_model.through.__name__ + "_set"
                    ).lower()
                if hasattr(obj, related_set_name):
                    related_model = getattr(obj, related_set_name)
            # clear real m2m
            if hasattr(related_model, "clear"):
                old_m2ms[model] = []
                # stock items in order to not recreate them
                for old_item in related_model.all():
                    old_m2ms[model].append(old_item)
                related_model.clear()
            else:
                for r in related_model.all():
                    r.delete()
        for key, value in m2m:
            related_model = getattr(obj, key + "s")
            related_data = {}  # used for intermediary models

            # an intermediary model is used
            if (
                hasattr(related_model, "through")
                and not related_model.through._meta.auto_created
            ):
                for field in related_model.through._meta.get_fields():
                    # is used for the obj or target
                    if getattr(field, "related_model", None) and (
                        field.related_model == related_model.model
                        or isinstance(obj, field.related_model)
                    ):
                        continue
                    if field.name in getattr(
                        related_model.through, "RELATED_ATTRS", []
                    ):
                        continue
                    related_data[field.name] = None

            if key not in m2m_items:
                if type(key) == dict:
                    vals = []
                    for item in related_model.all():
                        v = {}
                        for k in value.keys():
                            v[k] = getattr(item, k)
                        vals.append(v)
                    m2m_items[key] = vals
                else:
                    m2m_items[key] = related_model.all()
            if value not in m2m_items[key]:
                if type(value) == dict:
                    model = related_model.model
                    if (
                        hasattr(related_model, "through")
                        and related_model.through
                        and hasattr(related_model.through, "RELATIVE_MODELS")
                        and self.get_saved_model()
                        in related_model.through.RELATIVE_MODELS
                    ):
                        # the form is dealing with the through parameter
                        model = related_model.through
                    # not m2m -> foreign key
                    if not hasattr(related_model, "clear"):
                        assert hasattr(
                            model, "MAIN_ATTR"
                        ), "Must define a MAIN_ATTR for " + str(model.__class__)
                        value[getattr(model, "MAIN_ATTR")] = obj

                    # check old links
                    my_old_item = None
                    if key in old_m2ms:
                        for old_item in old_m2ms[key]:
                            is_ok = True
                            for k in value:
                                new_val = value[k]
                                val_empty = False
                                if k.startswith("__empty-"):
                                    val_empty = True
                                    k = k[len("__empty-") :]
                                if not hasattr(old_item, k):
                                    continue
                                old_val = getattr(old_item, k)
                                if (
                                    val_empty and (old_val is None or old_val == "")
                                ) or old_val != new_val:
                                    is_ok = False
                                    break
                            if is_ok:
                                my_old_item = old_item
                                break
                    if my_old_item:
                        value = my_old_item
                    else:
                        if issubclass(model, models.BaseHistorizedItem):
                            value["history_modifier"] = self.request.user

                        get_or_create = False
                        if (
                            hasattr(model, "RELATIVE_MODELS")
                            and self.get_saved_model() in model.RELATIVE_MODELS
                        ):
                            value[model.RELATIVE_MODELS[self.get_saved_model()]] = obj
                            get_or_create = True

                        # check if there is no missing fields
                        # should be managed normally in forms but...
                        fields = model._meta.get_fields()

                        has_problematic_null = False
                        for field in fields:
                            if (
                                (field.name not in value or not value[field.name])
                                and (hasattr(field, "null") and not field.null)
                                and (hasattr(field, "blank") and not field.blank)
                                and (
                                    hasattr(field, "default")
                                    and (
                                        not field.default
                                        or field.default == NOT_PROVIDED
                                    )
                                )
                            ):
                                has_problematic_null = True
                                break
                        if has_problematic_null:
                            continue

                        if hasattr(model, "data") and "data" not in value:
                            value["data"] = {}

                        # remove intermediary model keys
                        for k in list(related_data.keys()):
                            if k in value:
                                related_data[k] = value.pop(k)

                        # extract m2m values - not suitable for creation
                        m2m_values = {}
                        for k in value.keys():
                            if type(value[k]) in (list, tuple):
                                m2m_values[k] = value.pop(k)

                        value = dict(
                            [
                                (k, value[k])
                                for k in value
                                if not k.startswith("__empty-")
                            ]
                        )
                        if get_or_create:
                            value, created = model.objects.get_or_create(**value)
                        else:
                            if "pk" in value and value["pk"]:
                                try:
                                    instance = model.objects.get(pk=value.pop("pk"))
                                except model.DoesNotExist:
                                    continue
                                for k in value:
                                    setattr(instance, k, value[k])
                                value = instance
                            else:
                                value = model.objects.create(**value)

                        for k in m2m_values:
                            setattr(value, k, m2m_values[k])
                        value.save()  # force post_save
                # check that an item is not add multiple times (forged forms)
                if value not in related_model.all() and (
                    not hasattr(related_model, "through")
                    or not isinstance(value, related_model.through)
                ):
                    # many to many and the value have been already managed

                    # an intermediary model is used
                    if (
                        hasattr(related_model, "through")
                        and not related_model.through._meta.auto_created
                    ):
                        for field in related_model.through._meta.get_fields():
                            if hasattr(field, "related_model") and field.related_model:
                                # assume that one foreign key is used for obj
                                # table and value table - more complex schema
                                # are not managed
                                if isinstance(value, field.related_model):
                                    related_data[field.name] = value
                                elif isinstance(obj, field.related_model):
                                    related_data[field.name] = obj
                        # let default value if is none
                        for k in list(related_data.keys()):
                            if related_data[k] is None:
                                related_data.pop(k)
                        related_model.through.objects.create(**related_data)
                    else:
                        related_model.add(value)
                # necessary to manage interaction between models like
                # material_index management for baseitems
                obj._cached_label_checked = False
                obj._post_saved_geo = False
                obj.save()

        # force post_save for old related m2ms (which can have been detached
        # from the current object)
        for model in old_m2ms:
            for item in old_m2ms[model]:
                # verify it hasn't been deleted
                q = item.__class__.objects.filter(pk=item.pk)
                if q.count():
                    item = q.all()[0]
                    item.skip_history_when_saving = True
                    item.save()

        if hasattr(obj, "fix"):
            # post save/m2m specific fix
            obj.fix()

        ishtaruser = (
            self.request.user.ishtaruser
            if hasattr(self.request.user, "ishtaruser")
            else None
        )
        if (
            ishtaruser
            and ishtaruser.current_profile
            and ishtaruser.current_profile.auto_pin
        ):
            # make the new object a default
            if self.current_obj_slug:
                self.request.session[self.current_obj_slug] = str(obj.pk)
            self.request.session[self.get_object_name(obj)] = str(obj.pk)
        dct = {"item": obj}
        self.current_object = obj
        self.post_save()

        if self.redirect_url:
            url = reverse(self.redirect_url)
            if self.open_created_in_redirect:
                url += "?open_item={}".format(obj.pk)
            res = HttpResponseRedirect(url)
            if not return_object:
                return res
        else:
            # force evaluation of lazy urls
            wizard_done_window = str(self.wizard_done_window)
            if wizard_done_window:
                dct["wizard_done_window"] = wizard_done_window
            res = render(self.request, self.wizard_done_template, dct)
        return return_object and (obj, res) or res

    def post_save(self):
        return

    def get_deleted(self, keys):
        """
        Get the deleted and non-deleted items in formsets
        """
        not_to_delete, to_delete = set(), set()
        for key in keys:
            items = key.split("-")
            if len(items) < 2 or items[-2] in to_delete:
                continue
            idx = items[-2]
            try:
                int(idx)
            except:
                continue
            if items[-1] == "DELETE":
                to_delete.add(idx)
                if idx in not_to_delete:
                    not_to_delete.remove(idx)
            elif idx not in not_to_delete:
                not_to_delete.add(idx)
        return (to_delete, not_to_delete)

    def get_form(self, step=None, data=None, files=None):
        # Manage formset
        if data:
            data = data.copy()
            if not step:
                step = self.steps.current
            try:
                form = self.get_form_list()[step]
            except KeyError:
                raise Http404()
            if hasattr(form, "management_form"):
                # manage deletion
                to_delete, not_to_delete = self.get_deleted(list(data.keys()))
                # raz deleted fields
                if self.formset_pop_deleted:
                    for key in list(data.keys()):
                        items = key.split("-")
                        if len(items) < 2 or items[-2] not in to_delete:
                            continue
                        data.pop(key)
                    if to_delete:
                        # reorganize
                        for idx, number in enumerate(
                            sorted(not_to_delete, key=lambda x: int(x))
                        ):
                            idx = str(idx)
                            if idx == number:
                                continue
                            for key in list(data.keys()):
                                items = key.split("-")
                                if len(items) > 2 and number == items[-2]:
                                    items[-2] = str(idx)
                                    k = "-".join(items)
                                    data[k] = data.pop(key)[0]
                # get a form key
                frm = form.form
                if callable(frm):
                    frm = frm(self.get_form_kwargs(step))

                total_field = 0
                if hasattr(frm, "count_valid_fields"):
                    total_field = frm.count_valid_fields(data)
                else:
                    required_fields = [
                        ki for ki in frm.fields if frm.fields[ki].required
                    ]
                    base_key = None
                    if required_fields:
                        base_key = required_fields[-1]
                    elif frm.fields.keys():
                        base_key = list(frm.fields.keys())[-1]
                    if base_key:
                        total_field = len(
                            [
                                key
                                for key in data.keys()
                                if base_key in key.split("-") and data[key]
                            ]
                        )
                init = self.get_form_initial(step, data=data)
                if (
                    init
                    and not to_delete
                    and (
                        not hasattr(self, "form_initialized")
                        or not self.form_initialized
                    )
                ):
                    total_field = max((total_field, len(init)))
                data[step + "-INITIAL_FORMS"] = str(total_field)
                data[step + "-TOTAL_FORMS"] = str(total_field + 1)
                # TODO:remove form_initialized?
                # update initialization
                # if request.POST and init and hasattr(self,
                #                                      'form_initialized') \
                #   and self.form_initialized:
                #    for k in init[0]:
                #        data[step + '-' + str(total_field) + '-' + k] = \
                #                                                     init[0][k]

        data = data or None
        form = super(Wizard, self).get_form(step, data, files)
        # add autofocus to first field
        frm = None
        if hasattr(form, "fields") and form.fields.keys():
            frm = form
        elif (
            hasattr(form, "extra_form")
            and hasattr(form.extra_form, "fields")
            and form.extra_form.fields.keys()
        ):
            frm = form.extra_form
        elif hasattr(form, "forms") and form.forms and form.forms[0].fields.keys():
            frm = form.forms[0]
        if frm:
            # autofocus on first field
            first_field = frm.fields[list(frm.fields.keys())[0]]
            attrs = first_field.widget.attrs
            attrs.update({"autofocus": "autofocus"})
            first_field.widget.attrs = attrs
            if not step:
                step = self.steps.current
            if self.filter_owns_items and self.filter_owns and step in self.filter_owns:
                for key in self.filter_owns[step]:
                    frm.fields[key].widget.source = (
                        str(frm.fields[key].widget.source) + "own/"
                    )
                    if frm.fields[key].widget.source_full is not None:
                        frm.fields[key].widget.source_full = (
                            str(frm.fields[key].widget.source_full) + "own/"
                        )
        return form

    def render_next_step(self, form, **kwargs):
        """
        Manage:
         - modify or delete button in formset: next step = current step
         - validate and end: nextstep = last step
        """
        request = self.request
        if (
            request.POST.get("formset_modify")
            or request.POST.get("formset_add")
            or (
                self.formset_pop_deleted
                and [
                    key
                    for key in request.POST.keys()
                    if key.endswith("DELETE") and request.POST[key]
                ]
            )
        ):
            return self.render(form)
        elif "validate_and_end" in request.POST and request.POST["validate_and_end"]:
            last_step = self.steps.last
            new_form = self.get_form(
                last_step,
                data=self.storage.get_step_data(last_step),
                files=self.storage.get_step_files(last_step),
            )
            self.storage.current_step = last_step
            return self.render(new_form)
        return super(Wizard, self).render_next_step(form, **kwargs)

    def post(self, *args, **kwargs):
        # manage previous (or next) step
        form_prev_step = self.request.POST.get("form_prev_step", None)
        if not form_prev_step:
            return super(Wizard, self).post(*args, **kwargs)
        try:
            # convert numerical step number to step name
            step_number = int(self.request.POST["form_prev_step"])
            wizard_goto_step = list(self.get_form_list().keys())[step_number]
        except (ValueError, IndexError):
            return super(Wizard, self).post(*args, **kwargs)
        self.storage.current_step = wizard_goto_step
        return redirect(self.get_step_url(wizard_goto_step))

    def session_get_keys(self, form_key):
        """Get list of available keys for a specific form"""
        request = self.request
        storage = self.storage
        test = (
            storage.prefix in request.session
            and "step_data" in request.session[storage.prefix]
            and form_key in request.session[storage.prefix]["step_data"]
        )
        if not test:
            return []
        return request.session[storage.prefix]["step_data"][form_key].keys()

    def session_has_key(self, form_key, key=None, multi=None):
        """Check if the session has value of a specific form and (if provided)
        of a key
        """
        request = self.request
        storage = self.storage
        test = (
            storage.prefix in request.session
            and "step_data" in request.session[storage.prefix]
            and form_key in request.session[storage.prefix]["step_data"]
        )
        if not key or not test:
            return test
        if multi:
            # only check if the first field is available
            key = key.startswith(form_key) and key or form_key + "-0-" + key
            if key in request.session[storage.prefix]["step_data"][form_key]:
                return True
        key = key.startswith(form_key) and key or form_key + "-" + key
        return key in request.session[storage.prefix]["step_data"][form_key]

    @classmethod
    def session_reset(cls, request, url_name):
        prefix = url_name + normalize_name(cls.__name__)
        storage = get_storage(
            cls.storage_name, prefix, request, getattr(cls, "file_storage", None)
        )
        storage.reset()

    @classmethod
    def session_set_value(cls, request, form_key, key, value, reset=False):
        prefix = form_key.split("-")[1] + normalize_name(cls.__name__)
        storage = get_storage(
            cls.storage_name, prefix, request, getattr(cls, "file_storage", None)
        )
        if reset:
            storage.reset()
        data = storage.get_step_data(form_key)
        if not data:
            data = MultiValueDict()
        key = key if key.startswith(form_key) else form_key + "-" + key
        data[key] = value
        storage.set_step_data(form_key, data)

    @classmethod
    def prepare_serialization(cls, data):
        """
        Image and file cannot be passed as object
        """
        for k in data:
            if isinstance(data[k], ImageFieldFile) or isinstance(data[k], FileField):
                try:
                    data[k] = data[k].path
                except ValueError:
                    data[k] = None

    def session_get_value(self, form_key, key, multi=False, multi_value=False):
        """Get the value of a specific form"""
        if not self.session_has_key(form_key, key, multi):
            return
        request = self.request
        storage = self.storage
        if not multi:
            key = key.startswith(form_key) and key or form_key + "-" + key
            val = request.session[storage.prefix]["step_data"][form_key][key]
            if type(val) in (list, tuple) and val:
                if multi_value:
                    return val
                val = val[0]
            elif multi_value:
                if val:
                    return [val]
                return []
            return val
        vals = []
        for k in request.session[storage.prefix]["step_data"][form_key]:
            if (
                k.startswith(form_key)
                and k.endswith(key)
                and request.session[storage.prefix]["step_data"][form_key][k]
            ):
                val = request.session[storage.prefix]["step_data"][form_key][k]
                number = int(k[len(form_key) : -len(key)].strip("-"))
                if type(val) in (list, tuple):
                    val = val[0]
                vals.append((number, val))
        # reorder list
        vals = [v for idx, v in sorted(vals)]
        return vals

    def get_current_object(self):
        """Get the current object for an instanced wizard"""
        current_obj = None
        for key in self.main_item_select_keys:
            main_form_key = key + self.url_name
            try:
                idx = int(
                    self.session_get_value(main_form_key, self.current_object_key)
                )
                current_obj = self.model.objects.get(pk=idx)
                break
            except (TypeError, ValueError, ObjectDoesNotExist):
                pass
        return current_obj

    def get_form_initial(self, step, data=None):
        current_obj = self.get_current_object()
        current_step = self.steps.current
        request = self.request

        step_is_main_select = bool(
            [k for k in self.main_item_select_keys if step.startswith(k)]
        )
        if (
            step_is_main_select
            and step in self.form_list
            and "pk" in self.form_list[step].associated_models
        ):
            model_name = self.form_list[step].associated_models["pk"].__name__.lower()
            if step == current_step:
                self.storage.reset()
            val = model_name in request.session and request.session[model_name]
            if val:
                return MultiValueDict({"pk": val})
        elif current_obj:
            return self.get_instanced_init(current_obj, step)

        current_form = self.form_list[current_step]
        initial = MultiValueDict()
        if hasattr(current_form, "currents"):
            for key in current_form.currents:
                model_name = current_form.currents[key].__name__.lower()
                val = model_name in request.session and request.session[model_name]
                if val:
                    initial[key] = val
        if not initial and hasattr(current_form, "base_fields"):
            for k in current_form.base_fields:
                if self.request.GET.get(k, None):
                    initial[k] = self.request.GET[k]
        if initial:
            return initial
        return super(Wizard, self).get_form_initial(step)

    def get_object_name(self, obj):
        obj_name = obj.__class__.__name__.lower()
        # prefer a specialized name if available
        prefixes = self.storage.prefix.split("_")
        if len(prefixes) > 1 and prefixes[-2].startswith(obj_name):
            obj_name = prefixes[-2]
        return obj_name

    def _get_instanced_init_for_form(self, obj, c_form):
        """
        Get initial data from an object: simple form
        """
        initial = MultiValueDict()

        # manage json field
        if (
            hasattr(obj, "data")
            and obj.data
            and hasattr(self, "json_fields")
            and getattr(c_form, "form_slug", None)
            and c_form.form_slug in self.json_fields
            and obj.data
        ):
            for key in self.json_fields[c_form.form_slug]:
                if not key.startswith("data__"):
                    continue
                json_keys = key[len("data__") :].split("__")
                value = obj.data
                for json_key in json_keys:
                    if json_key not in value:
                        value = None
                        break
                    value = value[json_key]
                if value:
                    initial[key] = value
                elif value is False:
                    initial[key] = "False"

        for base_field in c_form.base_fields.keys():
            value = obj
            base_model = None
            if hasattr(c_form, "base_model") and base_field == c_form.base_model:
                base_model = base_field
            if hasattr(c_form, "base_models") and base_field in c_form.base_models:
                base_model = base_field
            if base_model:
                key = base_model + "s"
                initial.setlist(
                    base_field, [str(val.pk) for val in getattr(value, key).all()]
                )
            else:
                fields = base_field.split("__")
                for field in fields:
                    if callable(value):
                        value = value()
                    if not hasattr(value, field) or getattr(value, field) is None:
                        value = obj
                        break
                    value = getattr(value, field)
            if hasattr(value, "all") and callable(value.all):
                if not value.count():
                    continue
                initial.setlist(base_field, [str(v.pk) for v in value.all()])
                continue
            if value == obj:
                continue
            if hasattr(value, "pk"):
                value = value.pk
            if (
                value in (True, False)
                or isinstance(value, ImageFieldFile)
                or isinstance(value, FileField)
            ):
                initial[base_field] = value
            elif value is not None:
                initial[base_field] = str(value)
        return initial

    @staticmethod
    def _get_vals_for_instanced_init_for_formset(field, child_obj, vals):
        if hasattr(child_obj, field):
            value = getattr(child_obj, field)
            if hasattr(value, "pk"):
                value = value.pk
            elif hasattr(value, "all"):
                vals.setlist(
                    field, [str(v.pk) for v in getattr(child_obj, field).all()]
                )
                return vals
            if value is not None:
                vals[field] = str(value)
        elif hasattr(child_obj, field + "s"):
            # M2M
            vals.setlist(
                field, [str(v.pk) for v in getattr(child_obj, field + "s").all()]
            )
        return vals

    def _get_instanced_init_for_formset(self, obj, current_step, c_form):
        """
        Get initial data from an object: formset
        """
        initial = []
        if hasattr(c_form.form, "base_model"):
            key = c_form.form.base_model + "s"
        else:
            key = current_step.split("-")[0]
        if not hasattr(obj, key):
            return initial
        keys = list(c_form.form.base_fields.keys())
        related = getattr(obj, key)
        # manage through
        through = False
        if hasattr(related, "through") and related.through:
            if hasattr(related.through, "RELATED_SET_NAME"):
                related_set_name = related.through.RELATED_SET_NAME
            else:
                related_set_name = str(related.through.__name__ + "_set").lower()
            if hasattr(obj, related_set_name):
                through = True
                related = getattr(obj, related_set_name)

        query = related
        if not through and not obj._meta.ordering:
            query = query.order_by("pk")
        # an intermediary model is used
        through_fields = []
        if through and not related.model._meta.auto_created:
            target_field = getattr(related.model, c_form.form.base_model, None)
            if target_field:
                related_model = target_field.field.related_model
                for field in related_model._meta.get_fields():
                    through_fields.append(field.name)
                through_fields.append("pk")

        for child_obj in query.all():
            if not keys:
                break
            vals = MultiValueDict()
            if len(keys) == 1:
                # only one field: must be the id of the object
                vals[keys[0]] = str(child_obj.pk)
            else:
                for field in keys:
                    vals = self._get_vals_for_instanced_init_for_formset(
                        field, child_obj, vals
                    )
                for field in through_fields:
                    related_obj = getattr(child_obj, c_form.form.base_model)
                    vals = self._get_vals_for_instanced_init_for_formset(
                        field, related_obj, vals
                    )
            if vals:
                initial.append(vals)
        return initial

    def get_instanced_init(self, obj, step=None):
        """
        Get initial data from an object
        """
        current_step = step or self.steps.current
        c_form = self.form_list[current_step]

        ishtaruser = (
            self.request.user.ishtaruser
            if hasattr(self.request.user, "ishtaruser")
            else None
        )
        if (
            ishtaruser
            and ishtaruser.current_profile
            and ishtaruser.current_profile.auto_pin
        ):
            # make the current object the default item for the session
            self.request.session[self.get_object_name(obj)] = str(obj.pk)

        initial = MultiValueDict()
        # posted data or already in session
        if self.request.POST or (
            step in self.request.session[self.storage.prefix]
            and self.request.session[self.storage.prefix]["step_data"][step]
        ):
            return initial

        if hasattr(c_form, "base_fields"):
            return self._get_instanced_init_for_form(obj, c_form)
        elif hasattr(c_form, "management_form"):
            return self._get_instanced_init_for_formset(obj, current_step, c_form)
        return initial


class SearchWizard(IshtarWizard):
    model = None
    label = ""
    modification = None  # True when the wizard modify an item
    storage_name = "formtools.wizard.storage.session.SessionStorage"

    def get_wizard_name(self):
        """
        As the class name can interfere when reused, use the url_name
        """
        return self.url_name

    def get_prefix(self, request, *args, **kwargs):
        """As the class name can interfere when reused prefix with the url_name"""
        return self.url_name + super(SearchWizard, self).get_prefix(
            request, *args, **kwargs
        )

    def get_template_names(self):
        templates = ["ishtar/wizard/search.html"]
        return templates

    def get_label(self):
        return self.label

    def get_context_data(self, form, **kwargs):
        context = super(SearchWizard, self).get_context_data(form)
        self.request.session["CURRENT_ACTION"] = self.get_wizard_name()
        current_step = self.steps.current
        bookmark = self.request.GET.get("bookmark", None)
        default_search_vector = None
        if bookmark and self.model:
            slug = self.model.SLUG
            if slug == "site":
                slug = "archaeologicalsite"
            app_label = self.model.__module__.split(".")[0]
            try:
                app_label = self.model.__module__.split(".")[0]
                sq = models.SearchQuery.objects.get(
                    pk=bookmark,
                    content_type__app_label=app_label,
                    content_type__model=slug,
                    profile__person__ishtaruser__user_ptr=self.request.user,
                )
                default_search_vector = sq.query.replace('"', "''")
            except models.SearchQuery.DoesNotExist:
                pass
        context.update(
            {
                "current_step": self.form_list[current_step],
                "is_search": True,
                "wizard_label": self.get_label(),
                "default_search_vector": default_search_vector,
            }
        )
        return context


class PersonSearch(SearchWizard):
    model = models.Person


class OrganizationSearch(SearchWizard):
    model = models.Organization


class DocumentSearch(SearchWizard):
    model = models.Document


class DeletionWizard(Wizard):
    def __init__(self, *args, **kwargs):
        if (not hasattr(self, "fields") or not self.fields) and (
            hasattr(self, "model") and hasattr(self.model, "TABLE_COLS")
        ):
            self.fields = self.model.TABLE_COLS
        assert self.model
        super(DeletionWizard, self).__init__(*args, **kwargs)

    def get_formated_datas(self, forms):
        datas = super(DeletionWizard, self).get_formated_datas(forms)
        self.current_obj = None
        for form in forms:
            if not hasattr(form, "cleaned_data"):
                continue
            for key in form.cleaned_data:
                if key == "pk":
                    model = form.associated_models["pk"]
                    self.current_obj = model.objects.get(pk=form.cleaned_data["pk"])
        if not self.current_obj:
            return datas
        res = {}
        for field in self.model._meta.get_fields():
            if field.name not in self.fields:
                continue
            value = getattr(self.current_obj, field.name)
            if not value:
                continue
            label = ""
            if hasattr(field, "verbose_name"):
                label = field.verbose_name
            if hasattr(value, "all"):
                if not label and hasattr(field, "related_model"):
                    label = field.related_model._meta.verbose_name_plural
                value = ", ".join([str(item) for item in value.all()])
                if not value:
                    continue
            else:
                value = str(value)
            res[field.name] = (label, value, "")
        if not datas and self.fields:
            datas = [["", []]]
        for field in self.fields:
            if field in res:
                datas[0][1].append(res[field])
        return datas

    def done(self, form_list, **kwargs):
        obj = self.get_current_object()
        if obj:
            try:
                obj.delete()
            except ObjectDoesNotExist:
                pass
        return render(self.request, "ishtar/wizard/wizard_delete_done.html", {})


class MultipleItemWizard(Wizard):
    main_item_select_keys = ("selec-",)
    current_object_key = "pks"

    def get_current_objects(self):
        current_objs = []
        for key in self.main_item_select_keys:
            main_form_key = key + self.url_name
            try:
                pks = self.session_get_value(main_form_key, self.current_object_key)
                if pks:
                    for pk in pks.split(","):
                        current_objs.append(self.model.objects.get(pk=int(pk)))
            except (TypeError, ValueError, ObjectDoesNotExist):
                pass
        return current_objs


class MultipleDeletionWizard(MultipleItemWizard):
    def __init__(self, *args, **kwargs):
        if (not hasattr(self, "fields") or not self.fields) and (
            hasattr(self, "model") and hasattr(self.model, "TABLE_COLS")
        ):
            self.fields = self.model.TABLE_COLS
        assert self.model
        super(MultipleDeletionWizard, self).__init__(*args, **kwargs)

    def get_template_names(self):
        current_step = self.steps.current
        if (
            current_step.startswith("final-")
            and current_step not in self.wizard_templates
        ):
            return ["ishtar/wizard/delete_wizard.html"]
        return super(MultipleDeletionWizard, self).get_template_names()

    def get_formated_datas(self, forms):
        datas = super(MultipleDeletionWizard, self).get_formated_datas(forms)
        self.current_objs = []
        for form in forms:
            if not hasattr(form, "cleaned_data"):
                continue
            for key in form.cleaned_data:
                if key == "pks":
                    model = form.associated_models["pks"]
                    pks = form.cleaned_data["pks"].split(",")
                    for pk in pks:
                        try:
                            self.current_objs.append(model.objects.get(pk=pk))
                        except model.DoesNotExist:
                            continue
        if not self.current_objs:
            return datas
        full_res = []
        for current_obj in self.current_objs:
            res = {}
            for field in self.model._meta.get_fields():
                if field.name not in self.fields:
                    continue
                value = getattr(current_obj, field.name)
                if not value:
                    continue
                label = ""
                if hasattr(field, "verbose_name"):
                    label = field.verbose_name
                if hasattr(value, "all"):
                    if not label and hasattr(field, "related_model"):
                        label = field.related_model._meta.verbose_name_plural
                    value = ", ".join([str(item) for item in value.all()])
                    if not value:
                        continue
                else:
                    value = str(value)
                res[field.name] = (label, value, "")
            full_res.append(res)
        if not datas and self.fields:
            datas = [["", []]]
        datas = []
        for idx, res in enumerate(full_res):
            data = []
            for field in self.fields:
                if field in res:
                    data.append(res[field])
            datas.append((str(self.current_objs[idx]), data))
        return datas

    def get_context_data(self, form, **kwargs):
        data = super(MultipleDeletionWizard, self).get_context_data(form, **kwargs)
        data["current_objs"] = self.get_current_objects()
        return data

    def done(self, form_list, **kwargs):
        objs = self.get_current_objects()
        lbls = []
        for obj in objs:
            lbls.append(str(obj))
            try:
                obj.delete()
            except ObjectDoesNotExist:
                pass
        msg = '<i class="fa fa-trash" aria-hidden="true"></i>&nbsp;&nbsp;'
        msg += str(_("{} deleted.")).format(" ; ".join(lbls))
        messages.add_message(self.request, messages.INFO, msg)
        if self.redirect_url:
            return HttpResponseRedirect(reverse(self.redirect_url))
        return render(self.request, "ishtar/wizard/wizard_delete_done.html", {})


class ClosingWizard(Wizard):
    # "close" an item
    # to be define in the overloaded class
    model = None
    fields = []

    def get_formated_datas(self, forms):
        datas = super(ClosingWizard, self).get_formated_datas(forms)
        self.current_obj = None
        for form in forms:
            if not hasattr(form, "cleaned_data"):
                continue
            for key in form.cleaned_data:
                if key == "pk":
                    model = form.associated_models["pk"]
                    self.current_obj = model.objects.get(pk=form.cleaned_data["pk"])
        if not self.current_obj:
            return datas
        res = {}
        for field in self.model._meta.fields + self.model._meta.many_to_many:
            if field.name not in self.fields:
                continue
            value = getattr(self.current_obj, field.name)
            if not value:
                continue
            if hasattr(value, "all"):
                value = ", ".join([str(item) for item in value.all()])
                if not value:
                    continue
            else:
                value = str(value)
            res[field.name] = (field.verbose_name, value, "")
        if not datas and self.fields:
            datas = [["", []]]
        for field in self.fields:
            if field in res:
                datas[0][1].append(res[field])
        return datas

    def done(self, form_list, **kwargs):
        obj = self.get_current_object()
        for form in form_list:
            if form.is_valid():
                if "end_date" in form.cleaned_data and hasattr(obj, "end_date"):
                    obj.end_date = form.cleaned_data["end_date"]
                    obj.save()
        return render(self.request, "ishtar/wizard/wizard_closing_done.html", {})


class PersonWizard(Wizard):
    model = models.Person
    wizard_templates = {"identity-person_creation": "ishtar/wizard/wizard_person.html"}
    wizard_done_window = reverse_lazy("show-person")
    redirect_url = "person_modification"


class PersonModifWizard(PersonWizard):
    modification = True
    wizard_templates = {
        "identity-person_modification": "ishtar/wizard/wizard_person.html"
    }


class PersonDeletionWizard(MultipleDeletionWizard):
    model = models.Person
    fields = model.TABLE_COLS
    redirect_url = "person_deletion"
    wizard_templates = {
        "final-person_deletion": "ishtar/wizard/wizard_person_deletion.html"
    }


class IshtarUserDeletionWizard(DeletionWizard):
    model = models.IshtarUser
    fields = model.TABLE_COLS


class OrganizationWizard(Wizard):
    model = models.Organization
    wizard_done_window = reverse_lazy("show-organization")
    redirect_url = "organization_modification"


class OrganizationModifWizard(OrganizationWizard):
    modification = True


class OrganizationDeletionWizard(MultipleDeletionWizard):
    model = models.Organization
    fields = model.TABLE_COLS
    redirect_url = "organization_deletion"
    wizard_templates = {
        "final-organization_deletion": "ishtar/wizard/wizard_organization_deletion.html"
    }


class AccountWizard(Wizard):
    model = models.Person
    formset_pop_deleted = False
    wizard_done_window = reverse_lazy("show-person")

    def get_formated_datas(self, forms):
        datas = super(AccountWizard, self).get_formated_datas(forms)
        for form in forms:
            if not hasattr(form, "cleaned_data"):
                continue
            for key in form.cleaned_data:
                if key == "hidden_password" and form.cleaned_data[key]:
                    datas[-1][1].append((_("New password"), "*" * 8))
        return datas

    def done(self, form_list, **kwargs):
        """
        Save the account
        """
        form_dict = kwargs["form_dict"]

        main_form = form_dict["account-account_management"]
        if not main_form.is_valid():
            return self.render(main_form)

        dct = {}
        associated_models = main_form.associated_models
        if type(main_form.cleaned_data) == dict:
            for key in main_form.cleaned_data:
                if key == "pk":
                    continue
                value = main_form.cleaned_data[key]
                if key in associated_models and value:
                    value = associated_models[key].objects.get(pk=value)
                dct[key] = value
        person = self.get_current_object()
        if not person:
            return self.render(form_dict["selec-account_management"])

        for key in list(dct.keys()):
            if key.startswith("hidden_password"):
                dct["password"] = dct.pop(key)
        try:
            account = models.IshtarUser.objects.get(person=person).user_ptr
            account.username = dct["username"]
            account.email = dct["email"]
        except ObjectDoesNotExist:
            now = datetime.datetime.now()
            account = models.User.objects.create(
                username=dct["username"],
                email=dct["email"],
                first_name=person.surname or "***",
                last_name=person.name or "***",
                is_staff=False,
                is_active=True,
                is_superuser=False,
                last_login=now,
                date_joined=now,
            )
            ishtaruser = account.ishtaruser
            old_person_pk = ishtaruser.person.pk
            ishtaruser.person = person
            ishtaruser.save()
            models.Person.objects.get(pk=old_person_pk).delete()

        if dct["password"]:
            account.set_password(dct["password"])
        account.save()

        profile_form = form_dict["profile-account_management"]
        for form in profile_form:
            data = form.cleaned_data
            profile = None
            if data.get("pk", None):
                try:
                    profile = models.UserProfile.objects.get(
                        pk=data["pk"], person=person
                    )
                except models.UserProfile.DoesNotExist:
                    continue
                if data.get("DELETE", None):
                    profile.delete()
                    continue
            elif data.get("DELETE", None):
                continue

            name = data.get("name", None)

            profile_type_id = data.get("profile_type", None)
            if not profile_type_id:
                continue
            try:
                profile_type = models.ProfileType.objects.get(pk=profile_type_id)
            except models.ProfileType.DoesNotExist:
                continue

            if not name:
                name = profile_type.label

            if profile:
                profile.name = name
                profile.profile_type = profile_type
                profile.save()
            else:
                profile = models.UserProfile.objects.create(
                    profile_type=profile_type, person=person, name=name
                )
            area_pks = data.get("area", None)
            areas = []
            if area_pks:
                try:
                    areas = [models.Area.objects.get(pk=pk) for pk in area_pks]
                except models.Area.DoesNotExist:
                    # something wrong is happening - doing nothing on areas
                    continue
            profile.areas.clear()
            for area in areas:
                profile.areas.add(area)

        final_form = form_dict["final-account_management"]
        if (
            settings.ADMINS
            and type(final_form.cleaned_data) == dict
            and final_form.cleaned_data.get("send_password", None)
        ):
            site = Site.objects.get_current()
            app_name = site and ("Ishtar - " + site.name) or "Ishtar"
            context = {
                "login": dct["username"],
                "password": dct["password"],
                "app_name": app_name,
                "site": site and site.domain or "",
                "scheme": self.request.scheme,
            }
            t = loader.get_template("account_activation_email.txt")
            msg = t.render(context, self.request)
            subject = _("[%(app_name)s] Account creation/modification") % {
                "app_name": app_name
            }
            send_mail(
                subject, msg, settings.ADMINS[0][1], [dct["email"]], fail_silently=True
            )
        res = render(
            self.request,
            self.wizard_done_template,
            {"wizard_done_window": str(self.wizard_done_window), "item": person},
        )
        return res

    def get_form_kwargs(self, step=None):
        kwargs = super(AccountWizard, self).get_form_kwargs(step)
        if step == "account-account_management":
            kwargs["person"] = self.get_current_object()
        return kwargs

    def get_form(self, step=None, data=None, files=None):
        """
        Display the "Send email" field if necessary
        """
        form = super(AccountWizard, self).get_form(step, data, files)
        if not hasattr(form, "is_hidden"):
            return form
        if self.session_get_value("account-account_management", "hidden_password"):
            form.is_hidden = False
        return form


class SourceWizard(Wizard):
    model = None

    def get_extra_model(self, dct, m2m, form_list):
        dct = super(SourceWizard, self).get_extra_model(dct, m2m, form_list)
        if "history_modifier" in dct:
            dct.pop("history_modifier")
        return dct


DOCUMENT_EXCLUDED = models.Document.RELATED_MODELS + [
    "id",
    "history_creator",
    "history_modifier",
    "search_vector",
    "imports",
    "last_modified",
    "document",
]


class DocumentDeletionWizard(MultipleDeletionWizard):
    model = models.Document
    fields = [
        f.name
        for f in models.Document._meta.get_fields()
        if f.name not in DOCUMENT_EXCLUDED
    ]
    fields += models.Document.RELATED_MODELS
    filter_owns = {"selec-document_deletion": ["pks"]}
    redirect_url = "document_deletion"