✨ Debug mode for imports

4 files changed, 50 insertions, 1 deletions

diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index 90cabf0b1..b4be3d297 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -741,6 +741,7 @@ class Importer(object):
     MAIN_GEO = False
     LINE_FORMAT = []
     OBJECT_CLS = None
+    DEBUG = False
     UNICITY_KEYS = []
     # if set only models inside this list can be created
     MODEL_CREATION_LIMIT = []
@@ -835,6 +836,7 @@ class Importer(object):
         self._pre_import_values = self.PRE_IMPORT_VALUES.copy()
         self.history_modifier = history_modifier
         self.output = output
+        self.debug = []
         if not self.history_modifier:
             if self.import_instance and self.import_instance.user:
                 self.history_modifier = self.import_instance.user.user_ptr
@@ -1189,6 +1191,10 @@ class Importer(object):
                 continue
             data = update_data(defaults, data)
         self.validity.append(c_row)
+        if self.DEBUG:
+            debug_data = copy.deepcopy(data)
+            debug_data["_debug_current_line"] = idx_line + 1
+            self.debug.append(debug_data)
         if not self.c_errors and (idx_col + 1) < self.min_col_number:
             self.c_errors = True
             self.errors.append(
diff --git a/ishtar_common/migrations/0242_import_debug_field.py b/ishtar_common/migrations/0242_import_debug_field.py
new file mode 100644
index 000000000..9d4ef5142
--- /dev/null
+++ b/ishtar_common/migrations/0242_import_debug_field.py
@@ -0,0 +1,23 @@
+# Generated by Django 2.2.24 on 2024-03-22 12:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ishtar_common', '0241_migrate_old_department'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='import',
+            name='debug',
+            field=models.TextField(blank=True, default='', verbose_name='Debug'),
+        ),
+        migrations.AddField(
+            model_name='importertype',
+            name='debug',
+            field=models.BooleanField(default=False, verbose_name='Debug'),
+        ),
+    ]
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index b25fdc684..51d678401 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -73,6 +73,7 @@ from ishtar_common.utils import (
     get_all_related_m2m_objects_with_model,
     get_session_var,
     import_class,
+    StrJSONEncoder,
     max_size_help,
     num2col,
     put_session_message,
@@ -211,6 +212,7 @@ class ImporterType(models.Model):
         help_text=_("If an error is encountered with the following character strings, the error is not reported in "
                     "the error file. Each message is separated with a line break.")
     )
+    debug = models.BooleanField(verbose_name=_("Debug"), default=False)
 
     objects = SlugModelManager()
     SERIALIZATION_EXCLUDE = ["users"]
@@ -395,6 +397,7 @@ class ImporterType(models.Model):
             "MODEL_CREATION_LIMIT": MODEL_CREATION_LIMIT,
             "TYPE": self.type,
             "MAIN_GEO": self.is_main_geometry,
+            "DEBUG": self.debug
         }
         name = str(
             "".join(
@@ -1977,6 +1980,7 @@ class Import(BaseImport):
     next_import = models.ForeignKey(
         "Import", blank=True, null=True, on_delete=models.SET_NULL,
         verbose_name=_("Next import"), related_name="imports")
+    debug = models.TextField(verbose_name=_("Debug"), blank=True, default="")
 
     class Meta:
         verbose_name = _("Import - Import")
@@ -2608,6 +2612,8 @@ class Import(BaseImport):
                 self.state = "PI"
             else:
                 self.state = "FE"
+            if importer.debug:
+                self.debug = json.dumps(importer.debug, cls=StrJSONEncoder, indent=2)
             self.save()
             if not return_importer_and_data:
                 return
@@ -2619,7 +2625,8 @@ class Import(BaseImport):
         self.result_file.save(
             result_file, ContentFile(importer.get_csv_result().encode("utf-8"))
         )
-
+        if importer.debug:
+            self.debug = json.dumps(importer.debug, cls=StrJSONEncoder, indent=2)
         if importer.errors:
             if line_to_process:
                 self.state = "PI"
diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py
index 08e7a49b4..c88342782 100644
--- a/ishtar_common/utils.py
+++ b/ishtar_common/utils.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # Copyright (C) 2013-2016 Étienne Loks  <etienne.loks_AT_peacefrogsDOTnet>
+import json
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -2557,6 +2558,18 @@ def human_date(value):
     return value.strftime(settings.DATE_FORMAT)
 
 
+class StrJSONEncoder(json.JSONEncoder):
+    def default(self, o):
+        try:
+            return super().default(o)
+        except TypeError:
+            s = f"<{o.__class__.__name__}> "
+            if hasattr(o, "pk"):
+                s += f"[{o.pk}] "
+            s += str(o)
+            return s
+
+
 class IshtarFileSystemStorage(FileSystemStorage):
     def exists(self, name):
         path_name = self.path(name)