summaryrefslogtreecommitdiff
path: root/ishtar_common
diff options
context:
space:
mode:
authorÉtienne Loks <etienne.loks@iggdrasil.net>2024-03-22 13:10:52 +0100
committerÉtienne Loks <etienne.loks@iggdrasil.net>2024-03-22 13:23:14 +0100
commita4b5f36facba9d2d8685c796acda7d4c35e3c51e (patch)
tree594e31ccbf84130a79565b70b54cbe5318e59d80 /ishtar_common
parentaf5f58021b975d2bd4081ac80cc9976f096d3a16 (diff)
downloadIshtar-a4b5f36facba9d2d8685c796acda7d4c35e3c51e.tar.bz2
Ishtar-a4b5f36facba9d2d8685c796acda7d4c35e3c51e.zip
✨ Debug mode for imports
Diffstat (limited to 'ishtar_common')
-rw-r--r--ishtar_common/data_importer.py6
-rw-r--r--ishtar_common/migrations/0242_import_debug_field.py23
-rw-r--r--ishtar_common/models_imports.py9
-rw-r--r--ishtar_common/utils.py13
4 files changed, 50 insertions, 1 deletions
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index 90cabf0b1..b4be3d297 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -741,6 +741,7 @@ class Importer(object):
MAIN_GEO = False
LINE_FORMAT = []
OBJECT_CLS = None
+ DEBUG = False
UNICITY_KEYS = []
# if set only models inside this list can be created
MODEL_CREATION_LIMIT = []
@@ -835,6 +836,7 @@ class Importer(object):
self._pre_import_values = self.PRE_IMPORT_VALUES.copy()
self.history_modifier = history_modifier
self.output = output
+ self.debug = []
if not self.history_modifier:
if self.import_instance and self.import_instance.user:
self.history_modifier = self.import_instance.user.user_ptr
@@ -1189,6 +1191,10 @@ class Importer(object):
continue
data = update_data(defaults, data)
self.validity.append(c_row)
+ if self.DEBUG:
+ debug_data = copy.deepcopy(data)
+ debug_data["_debug_current_line"] = idx_line + 1
+ self.debug.append(debug_data)
if not self.c_errors and (idx_col + 1) < self.min_col_number:
self.c_errors = True
self.errors.append(
diff --git a/ishtar_common/migrations/0242_import_debug_field.py b/ishtar_common/migrations/0242_import_debug_field.py
new file mode 100644
index 000000000..9d4ef5142
--- /dev/null
+++ b/ishtar_common/migrations/0242_import_debug_field.py
@@ -0,0 +1,23 @@
+# Generated by Django 2.2.24 on 2024-03-22 12:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('ishtar_common', '0241_migrate_old_department'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='import',
+ name='debug',
+ field=models.TextField(blank=True, default='', verbose_name='Debug'),
+ ),
+ migrations.AddField(
+ model_name='importertype',
+ name='debug',
+ field=models.BooleanField(default=False, verbose_name='Debug'),
+ ),
+ ]
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index b25fdc684..51d678401 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -73,6 +73,7 @@ from ishtar_common.utils import (
get_all_related_m2m_objects_with_model,
get_session_var,
import_class,
+ StrJSONEncoder,
max_size_help,
num2col,
put_session_message,
@@ -211,6 +212,7 @@ class ImporterType(models.Model):
help_text=_("If an error is encountered with the following character strings, the error is not reported in "
"the error file. Each message is separated with a line break.")
)
+ debug = models.BooleanField(verbose_name=_("Debug"), default=False)
objects = SlugModelManager()
SERIALIZATION_EXCLUDE = ["users"]
@@ -395,6 +397,7 @@ class ImporterType(models.Model):
"MODEL_CREATION_LIMIT": MODEL_CREATION_LIMIT,
"TYPE": self.type,
"MAIN_GEO": self.is_main_geometry,
+ "DEBUG": self.debug
}
name = str(
"".join(
@@ -1977,6 +1980,7 @@ class Import(BaseImport):
next_import = models.ForeignKey(
"Import", blank=True, null=True, on_delete=models.SET_NULL,
verbose_name=_("Next import"), related_name="imports")
+ debug = models.TextField(verbose_name=_("Debug"), blank=True, default="")
class Meta:
verbose_name = _("Import - Import")
@@ -2608,6 +2612,8 @@ class Import(BaseImport):
self.state = "PI"
else:
self.state = "FE"
+ if importer.debug:
+ self.debug = json.dumps(importer.debug, cls=StrJSONEncoder, indent=2)
self.save()
if not return_importer_and_data:
return
@@ -2619,7 +2625,8 @@ class Import(BaseImport):
self.result_file.save(
result_file, ContentFile(importer.get_csv_result().encode("utf-8"))
)
-
+ if importer.debug:
+ self.debug = json.dumps(importer.debug, cls=StrJSONEncoder, indent=2)
if importer.errors:
if line_to_process:
self.state = "PI"
diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py
index f1c899a69..1ebb87565 100644
--- a/ishtar_common/utils.py
+++ b/ishtar_common/utils.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright (C) 2013-2016 Étienne Loks <etienne.loks_AT_peacefrogsDOTnet>
+import json
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -2482,6 +2483,18 @@ def human_date(value):
return value.strftime(settings.DATE_FORMAT)
+class StrJSONEncoder(json.JSONEncoder):
+ def default(self, o):
+ try:
+ return super().default(o)
+ except TypeError:
+ s = f"<{o.__class__.__name__}> "
+ if hasattr(o, "pk"):
+ s += f"[{o.pk}] "
+ s += str(o)
+ return s
+
+
class IshtarFileSystemStorage(FileSystemStorage):
def exists(self, name):
path_name = self.path(name)