🐛 CSV result - fix missing columns (refs #5677) - 🩹 improve CSV result values

4 files changed, 37 insertions, 8 deletions

diff --git a/changelog/en/changelog_2022-06-15.md b/changelog/en/changelog_2022-06-15.md
index a17fc9080..b0486623c 100644
--- a/changelog/en/changelog_2022-06-15.md
+++ b/changelog/en/changelog_2022-06-15.md
@@ -43,6 +43,10 @@ v4.0.XX - 2099-12-31
 - geo sheet:
   - do not zoom or select point with no x, y coordinates
   - fix 2d point display
+- imports - CSV result:
+  - fix missing columns (#5677)
+  - improve values format
+
 
 v4.0.75 - 2024-04-16
 --------------------
diff --git a/changelog/fr/changelog_2023-01-25.md b/changelog/fr/changelog_2023-01-25.md
index 9d8d3f0f6..19344b32f 100644
--- a/changelog/fr/changelog_2023-01-25.md
+++ b/changelog/fr/changelog_2023-01-25.md
@@ -45,6 +45,9 @@ v4.0.XX - 2099-12-31
 - fiche géo :
   - retrait des fonctions de zoom et de sélection pour les points sans coordonnées x et y
   - correction de l'affichage de points 2D
+- imports - CSV de résultat :
+  - correction de colonnes manquantes (#5677)
+  - amélioration du formatage des valeurs
 
 v4.0.75 - 2024-04-16
 --------------------
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index 3b7e4305e..90cabf0b1 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -61,7 +61,7 @@ def pre_importer_action(func):
     return wrapper
 
 
-class ImportFormater(object):
+class ImportFormater:
     def __init__(
         self,
         field_name,
@@ -1139,7 +1139,6 @@ class Importer(object):
         self.to_be_close = []
         self.idx_line = idx_line
         if self.skip_lines > idx_line:
-            self.validity.append(line)
             return
         if not line:
             self.validity.append([])
@@ -1189,7 +1188,6 @@ class Importer(object):
             if nok or not any(1 for k in test_dict if test_dict[k]):  # test empty dict
                 continue
             data = update_data(defaults, data)
-
         self.validity.append(c_row)
         if not self.c_errors and (idx_col + 1) < self.min_col_number:
             self.c_errors = True
@@ -1445,7 +1443,7 @@ class Importer(object):
             func = formater.formater
             if type(func) in (list, tuple):
                 func = func[idx_fields]
-            if not callable(func) and type(func) == str:
+            if not callable(func) and isinstance(func, str):
                 func = getattr(self, func)
 
             values = [val]
@@ -1523,18 +1521,21 @@ class Importer(object):
             printed_values = value
             if type(value) not in (list, tuple):
                 printed_values = [value]
+            if isinstance(func, FileFormater):
+                printed_values = [str(v).split(os.sep)[-1] for v in printed_values]
             try:
                 # don't reunicode - unicoded values
                 c_values.append(" ; ".join([v for v in printed_values]))
             except TypeError:
                 c_values.append(" ; ".join([str(v) for v in printed_values]))
+
             if value is None and formater.required:
                 self.c_errors = True
                 self.errors.append(
                     (idx_line + 1, idx_col + 1, self.ERRORS["value_required"])
                 )
+                c_row.append("")
                 return data
-
             field_names = [field_name]
             force_news = [force_new]
             concats = formater.concat
@@ -1557,6 +1558,8 @@ class Importer(object):
                 self._throughs.append((formater, value))
             else:
                 for idx, f_name in enumerate(field_names):
+                    if not f_name:
+                        continue
                     self._field_name_to_data_dict(
                         f_name,
                         value,
@@ -2201,7 +2204,8 @@ class Importer(object):
         return self._get_csv(self.errors, header=[_("line"), _("col"), _("error")])
 
     def get_csv_result(self):
-        return self._get_csv(self.validity)
+        header = [(line_format and line_format.label) or "-" for line_format in self.LINE_FORMAT]
+        return self._get_csv(self.validity, header=header)
 
     def get_csv_matches(self):
         header = [_("field"), _("source"), _("result")]
diff --git a/ishtar_common/views.py b/ishtar_common/views.py
index c96876713..4fc2c94b7 100644
--- a/ishtar_common/views.py
+++ b/ishtar_common/views.py
@@ -2285,7 +2285,8 @@ class ImportCSVView(ImportPermissionMixin, IshtarMixin, LoginRequiredMixin, Temp
         user = self.request.user
         if not user.pk:
             raise Http404()
-        model = models.ImportGroup if kwargs.get("group", None) else models.Import
+        self.is_group = kwargs.get("group", None)
+        model = models.ImportGroup if self.is_group else models.Import
         q = model.query_can_access(self.request.user, perm=self.permission_full).filter(pk=kwargs.get("pk", -1))
         if not q.count():
             raise Http404()
@@ -2316,13 +2317,30 @@ class ImportCSVView(ImportPermissionMixin, IshtarMixin, LoginRequiredMixin, Temp
                 import_item=self.import_item).values_list("pk", "ignored")
         data["has_line_errors"] = has_line_errors
         data["content"] = []
+        header = []
         delimiter = ","
         if self.kwargs["target"] == "source":
             delimiter = self.import_item.csv_sep
+            if not self.is_group:
+                q = models.ImporterColumn.objects.filter(importer_type=self.import_item.importer_type).order_by("col_number")
+                cols = list(q.values_list("col_number", "label"))
+                col_dict = dict(cols)
+                if cols:
+                    for idx in range(cols[-1][0]):
+                        if (idx + 1) in col_dict:
+                            header.append(col_dict[idx + 1])
+                        else:
+                            header.append("-")
+                    data["header"] = header
         with open(self.csv_file.path, "r", encoding=encoding) as f:
             reader = csv.reader(f, delimiter=delimiter)
             for idx, line in enumerate(reader):
-                if not idx:
+                if self.kwargs["target"] == "source":
+                    if idx < self.import_item.skip_lines:
+                        if not header:
+                            data["header"] = line
+                        continue
+                elif not idx:
                     data["header"] = line
                     continue
                 if line_errors and len(line_errors) >= idx: