✨ Pre-import form: manage import

4 files changed, 138 insertions, 45 deletions

diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index 6a58e0409..d5e6f966c 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -748,6 +748,7 @@ class Importer(object):
     MODEL_CREATION_LIMIT = []
     EXTRA_DEFAULTS = {}
     DEFAULTS = {}
+    PRE_IMPORT_VALUES = {}  # values from a form before the import
     ERRORS = {
         "header_check": _(
             "The given file is not correct. Check the file "
@@ -839,6 +840,7 @@ class Importer(object):
                 if k not in self._defaults:
                     self._defaults[k] = {}
                 self._defaults[k].update(self.EXTRA_DEFAULTS[k])
+        self._pre_import_values = self.PRE_IMPORT_VALUES.copy()
         self.history_modifier = history_modifier
         self.output = output
         if not self.history_modifier:
@@ -1151,7 +1153,7 @@ class Importer(object):
         idx_col = 0
         for idx_col, val in enumerate(line):
             try:
-                self._row_processing(c_row, idx_col, idx_line, val, data)
+                data = self._row_processing(c_row, idx_col, idx_line, val, data)
                 # nosec: no catch to force continue processing of lines
             except:  # nosec
                 pass
@@ -1367,7 +1369,7 @@ class Importer(object):
 
     def _row_processing(self, c_row, idx_col, idx_line, val, data):
         if idx_col >= len(self.line_format):
-            return
+            return data
 
         formater = self.line_format[idx_col]
 
@@ -1376,7 +1378,7 @@ class Importer(object):
 
         if not formater or not formater.field_name:
             c_row.append(_("Not imported"))
-            return
+            return data
 
         if formater.regexp:
             # multiline regexp is a mess...
@@ -1390,7 +1392,7 @@ class Importer(object):
                     self.c_errors = True
                 elif not val.strip():
                     c_row.append("")
-                    return
+                    return data
                 val = val.replace(NEW_LINE_BREAK, "\n")
                 self.errors.append(
                     (
@@ -1400,7 +1402,7 @@ class Importer(object):
                     )
                 )
                 c_row.append("")
-                return
+                return data
             val_group = []
             for g in formater.regexp.findall(val):
                 if isinstance(g, (tuple, list)):
@@ -1470,7 +1472,7 @@ class Importer(object):
                         self.c_errors = True
                     self.errors.append((idx_line + 1, idx_col + 1, str(e)))
                     c_values.append("")
-                    return
+                    return data
                 if formater.value_format and value is not None and value != "":
                     if "{item" in formater.value_format:
                         value = formater.value_format.format(item=value)
@@ -1503,7 +1505,7 @@ class Importer(object):
                 self.errors.append(
                     (idx_line + 1, idx_col + 1, self.ERRORS["value_required"])
                 )
-                return
+                return data
 
             field_names = [field_name]
             force_news = [force_new]
@@ -1536,7 +1538,9 @@ class Importer(object):
                         concat=concats[idx],
                         concat_str=concat_str[idx],
                     )
+        data = update_data(self._pre_import_values, data)
         c_row.append(" ; ".join([v for v in c_values]))
+        return data
 
     def _get_field_m2m(
         self, attribute, data, c_path, new_created, field_object, idx_line=None
diff --git a/ishtar_common/forms_common.py b/ishtar_common/forms_common.py
index 3e3488ae9..cc61f4da7 100644
--- a/ishtar_common/forms_common.py
+++ b/ishtar_common/forms_common.py
@@ -671,19 +671,9 @@ class PreImportForm(IshtarForm):
                 attrs["widget"] = getattr(forms, widget_name)
             key = f"col_{- column.col_number}"
             form_field = getattr(forms, field_name)
-            if field_name == "ChoiceField":
-                if not target.formater_type.options:
-                    continue
-                options = target.formater_type.options.split(".")
-                if len(options) == 1:
-                    app = "ishtar_common"
-                else:
-                    app = options[0]
-                model_name = options[-1]
-                try:
-                    model = apps.get_model(app, model_name)
-                except LookupError:
-                    continue
+
+            model = target.formater_type.associated_model
+            if model:
                 if target.formater_type.many_split:
                     form_field = widgets.Select2MultipleField
                 self._types.append(
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index 454bed7e8..c03dd208b 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -65,14 +65,16 @@ from ishtar_common.model_managers import SlugModelManager
 
 from ishtar_common.utils import (
     create_slug,
+    generate_dict_from_list,
     get_all_related_m2m_objects_with_model,
-    put_session_message,
-    put_session_var,
     get_session_var,
-    num2col,
-    max_size_help,
     import_class,
+    max_size_help,
+    num2col,
+    put_session_message,
+    put_session_var,
     reverse_coordinates,
+    update_data
 )
 from ishtar_common.data_importer import (
     Importer,
@@ -237,7 +239,7 @@ class ImporterType(models.Model):
             return
         col_number = 1  # user number so we start with 1
         lst_col_number = 0
-        for column in self.columns.order_by("col_number").all():
+        for column in self.columns.filter(col_number__gt=0).order_by("col_number").all():
             while column.col_number > col_number:
                 col_number += 1
             # header
@@ -291,10 +293,13 @@ class ImporterType(models.Model):
         DEFAULTS = dict(
             [(default.keys, default.values) for default in self.defaults.all()]
         )
+        PRE_IMPORT_VALUES = {}
+        if import_instance:
+            PRE_IMPORT_VALUES = import_instance.pre_import_values
         LINE_FORMAT = []
         LINE_EXPORT_FORMAT = []
         idx = 0
-        for column in self.columns.order_by("col_number").all():
+        for column in self.columns.filter(col_number__gt=0).order_by("col_number").all():
             idx += 1
             while column.col_number > idx:
                 LINE_FORMAT.append(None)
@@ -354,6 +359,7 @@ class ImporterType(models.Model):
             "OBJECT_CLS": OBJECT_CLS,
             "DESC": self.description,
             "DEFAULTS": DEFAULTS,
+            "PRE_IMPORT_VALUES": PRE_IMPORT_VALUES,
             "LINE_FORMAT": LINE_FORMAT,
             "UNICITY_KEYS": UNICITY_KEYS,
             "LINE_EXPORT_FORMAT": LINE_EXPORT_FORMAT,
@@ -1129,6 +1135,26 @@ class FormaterType(models.Model):
     def natural_key(self):
         return self.formater_type, self.options, self.many_split
 
+    @property
+    def associated_model(self):
+        if hasattr(self, "__associated_model"):
+            return self.__associated_model
+        if self.formater_type != "TypeFormater" or not self.options:
+            self.__associated_model = None
+            return
+        self.__associated_model = None
+        options = self.options.split(".")
+        if len(options) == 1:
+            app = "ishtar_common"
+        else:
+            app = options[0]
+        model_name = options[-1]
+        try:
+            self.__associated_model = apps.get_model(app, model_name)
+        except LookupError:
+            pass
+        return self.__associated_model
+
     def __str__(self):
         return " - ".join(
             [
@@ -1331,6 +1357,14 @@ class BaseImport(models.Model):
     class Meta:
         abstract = True
 
+    @property
+    def has_pre_import_form(self) -> bool:
+        raise NotImplemented()
+
+    @property
+    def pre_import_form_is_valid(self) -> bool:
+        raise NotImplemented()
+
 
 class ImportGroup(BaseImport):
     importer_type = models.ForeignKey(ImporterGroup, on_delete=models.CASCADE,
@@ -1353,6 +1387,10 @@ class ImportGroup(BaseImport):
         return False
 
     @property
+    def pre_import_form_is_valid(self) -> bool:
+        return not any(-1 for imp in self.imports.all() if not imp.pre_import_form_is_valid)
+
+    @property
     def import_id(self):
         return f"group-{self.id}"
 
@@ -1386,7 +1424,8 @@ class ImportGroup(BaseImport):
             actions.append(("A", _("Analyse")))
         if self.state == "A":
             actions.append(("A", _("Re-analyse")))
-            actions.append(("I", _("Launch import")))
+            if not any(-1 for imp in self.import_list() if not imp.pre_import_form_is_valid):
+                actions.append(("I", _("Launch import")))
         if self.state in ("F", "FE"):
             actions.append(("A", _("Re-analyse")))
             actions.append(("I", _("Re-import")))
@@ -1614,6 +1653,52 @@ class Import(BaseImport):
                 return False
         return True
 
+    @property
+    def pre_import_values(self) -> dict:
+        values = {}
+        for column in self.importer_type.columns.filter(col_number__lte=0):
+            q = ImportColumnValue.objects.filter(column=column, import_item=self)
+            if not q.count():
+                continue
+            column_value = q.all()[0]
+            q = column_value.column.targets.all()
+            if not q.count():
+                continue
+            target = q.all()[0]
+
+            value = column_value.value
+            if value == "":
+                continue
+            many = target.formater_type.many_split
+            if many:
+                if value.startswith("['") and value.endswith("']"):
+                    value = value[2:-2].split("', '")
+            associated_model = target.formater_type.associated_model
+            if associated_model:
+                if many:
+                    try:
+                        value = [associated_model.objects.get(pk=int(pk)) for pk in value]
+                    except (associated_model.DoesNotExist, ValueError):
+                        continue
+                else:
+                    try:
+                        value = associated_model.objects.get(pk=value)
+                    except (associated_model.DoesNotExist, ValueError):
+                        continue
+
+            keys = target.target.split("__")
+            dct = generate_dict_from_list(keys, value)
+            values = update_data(values, dct)
+
+            q = column_value.column.duplicate_fields.all()
+            if not q.count():
+                continue
+            for dup in q.all():
+                keys = dup.field_name.split("__")
+                dct = generate_dict_from_list(keys, value)
+                values = update_data(values, dct)
+        return values
+
     def get_number_of_lines(self):
         if self.number_of_line:
             return self.number_of_line
@@ -1704,7 +1789,8 @@ class Import(BaseImport):
             actions.append(("A", _("Analyse")))
         if self.state in ("A", "PI"):
             actions.append(("A", _("Re-analyse")))
-            actions.append(("I", _("Launch import")))
+            if self.pre_import_form_is_valid:
+                actions.append(("I", _("Launch import")))
             if profile.experimental_feature:
                 if self.changed_checked:
                     actions.append(("IS", _("Step by step import")))
diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py
index 44d42f00f..9198e8d2c 100644
--- a/ishtar_common/utils.py
+++ b/ishtar_common/utils.py
@@ -273,32 +273,45 @@ def check_model_access_control(request, model, available_perms=None):
     return allowed, own
 
 
-def update_data(data_1, data_2, merge=False):
+def update_data(data, new_data, merge=False):
     """
     Update a data directory taking account of key detail
     """
     res = {}
-    if not isinstance(data_1, dict) or not isinstance(data_2, dict):
-        if data_2 and not data_1:
-            return data_2
+    if not isinstance(data, dict) or not isinstance(new_data, dict):
+        if new_data and not data:
+            return new_data
         if not merge:
-            if data_2:
-                return data_2
-            return data_1
-        if data_2 and data_2 != data_1:
-            return data_1 + " ; " + data_2
-        return data_1
-    for k in data_1:
-        if k not in data_2:
-            res[k] = data_1[k]
+            if new_data:
+                return new_data
+            return data
+        if new_data and data_2 != data:
+            return data + " ; " + new_data
+        return data
+    for k in data:
+        if k not in new_data:
+            res[k] = data[k]
         else:
-            res[k] = update_data(data_1[k], data_2[k], merge=merge)
-    for k in data_2:
-        if k not in data_1:
-            res[k] = data_2[k]
+            res[k] = update_data(data[k], new_data[k], merge=merge)
+    for k in new_data:
+        if k not in data:
+            res[k] = new_data[k]
     return res
 
 
+def generate_dict_from_list(lst: list, value) -> dict:
+    """
+    ("key1", "key2", "key3"), value -> {"key1": {"key2": {"key3": value}}}
+    """
+    dct = {}
+    for key in reversed(lst):
+        if not dct:
+            dct = {key: value}
+        else:
+            dct = {key: dct}
+    return dct
+
+
 def move_dict_data(data, key1, key2):
     """
     Move key1 value to key2 value in a data dict