summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorÉtienne Loks <etienne.loks@iggdrasil.net>2023-10-23 12:36:44 +0200
committerÉtienne Loks <etienne.loks@iggdrasil.net>2024-04-16 16:38:32 +0200
commit94a4a57148d2c81dbbc1251b7e79ba09282af217 (patch)
treee37714d36079e8af4d8e972d16859c6bc338b406
parent406252c203e6863aa4ae02619037f13439706f98 (diff)
downloadIshtar-94a4a57148d2c81dbbc1251b7e79ba09282af217.tar.bz2
Ishtar-94a4a57148d2c81dbbc1251b7e79ba09282af217.zip
✨ imports: allow errors to be pointed out line by line
-rw-r--r--archaeological_context_records/migrations/0113_contextrecord_imports_updated.py4
-rw-r--r--archaeological_files/migrations/0112_file_imports_updated.py4
-rw-r--r--archaeological_finds/migrations/0113_auto_20231024_1050.py (renamed from archaeological_finds/migrations/0113_auto_20231018_1553.py)4
-rw-r--r--archaeological_operations/migrations/0113_auto_20231024_1045.py (renamed from archaeological_operations/migrations/0113_auto_20231018_1551.py)4
-rw-r--r--archaeological_warehouse/migrations/0119_auto_20231024_1050.py (renamed from archaeological_warehouse/migrations/0119_auto_20231018_1553.py)4
-rw-r--r--ishtar_common/migrations/0230_auto_20231024_1045.py (renamed from ishtar_common/migrations/0230_auto_20231018_1551.py)17
-rw-r--r--ishtar_common/migrations/0231_default_mandatory_keys.py2
-rw-r--r--ishtar_common/models.py2
-rw-r--r--ishtar_common/models_imports.py58
-rw-r--r--ishtar_common/static/js/ishtar.js23
-rw-r--r--ishtar_common/templates/ishtar/blocks/view_import_csv.html14
-rw-r--r--ishtar_common/urls.py5
-rw-r--r--ishtar_common/views.py33
-rw-r--r--scss/custom.scss4
14 files changed, 156 insertions, 22 deletions
diff --git a/archaeological_context_records/migrations/0113_contextrecord_imports_updated.py b/archaeological_context_records/migrations/0113_contextrecord_imports_updated.py
index 6c9a5e085..b68f81937 100644
--- a/archaeological_context_records/migrations/0113_contextrecord_imports_updated.py
+++ b/archaeological_context_records/migrations/0113_contextrecord_imports_updated.py
@@ -1,4 +1,4 @@
-# Generated by Django 2.2.24 on 2023-10-18 15:51
+# Generated by Django 2.2.24 on 2023-10-24 10:45
from django.db import migrations, models
@@ -6,7 +6,7 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
- ('ishtar_common', '0230_auto_20231018_1551'),
+ ('ishtar_common', '0230_auto_20231024_1045'),
('archaeological_context_records', '0112_migrate_created'),
]
diff --git a/archaeological_files/migrations/0112_file_imports_updated.py b/archaeological_files/migrations/0112_file_imports_updated.py
index d227e0487..ab076fbc8 100644
--- a/archaeological_files/migrations/0112_file_imports_updated.py
+++ b/archaeological_files/migrations/0112_file_imports_updated.py
@@ -1,4 +1,4 @@
-# Generated by Django 2.2.24 on 2023-10-18 15:53
+# Generated by Django 2.2.24 on 2023-10-24 10:50
from django.db import migrations, models
@@ -6,7 +6,7 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
- ('ishtar_common', '0230_auto_20231018_1551'),
+ ('ishtar_common', '0231_default_mandatory_keys'),
('archaeological_files', '0111_migrate_created'),
]
diff --git a/archaeological_finds/migrations/0113_auto_20231018_1553.py b/archaeological_finds/migrations/0113_auto_20231024_1050.py
index e9bfad7fd..a1ede77e6 100644
--- a/archaeological_finds/migrations/0113_auto_20231018_1553.py
+++ b/archaeological_finds/migrations/0113_auto_20231024_1050.py
@@ -1,4 +1,4 @@
-# Generated by Django 2.2.24 on 2023-10-18 15:53
+# Generated by Django 2.2.24 on 2023-10-24 10:50
from django.db import migrations, models
@@ -6,7 +6,7 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
- ('ishtar_common', '0230_auto_20231018_1551'),
+ ('ishtar_common', '0231_default_mandatory_keys'),
('archaeological_finds', '0112_migrate_created'),
]
diff --git a/archaeological_operations/migrations/0113_auto_20231018_1551.py b/archaeological_operations/migrations/0113_auto_20231024_1045.py
index 11d1143d0..bb0b8ac0e 100644
--- a/archaeological_operations/migrations/0113_auto_20231018_1551.py
+++ b/archaeological_operations/migrations/0113_auto_20231024_1045.py
@@ -1,4 +1,4 @@
-# Generated by Django 2.2.24 on 2023-10-18 15:51
+# Generated by Django 2.2.24 on 2023-10-24 10:45
from django.db import migrations, models
@@ -6,7 +6,7 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
- ('ishtar_common', '0230_auto_20231018_1551'),
+ ('ishtar_common', '0230_auto_20231024_1045'),
('archaeological_operations', '0112_migrate_created'),
]
diff --git a/archaeological_warehouse/migrations/0119_auto_20231018_1553.py b/archaeological_warehouse/migrations/0119_auto_20231024_1050.py
index b84564711..5b908736d 100644
--- a/archaeological_warehouse/migrations/0119_auto_20231018_1553.py
+++ b/archaeological_warehouse/migrations/0119_auto_20231024_1050.py
@@ -1,4 +1,4 @@
-# Generated by Django 2.2.24 on 2023-10-18 15:53
+# Generated by Django 2.2.24 on 2023-10-24 10:50
from django.db import migrations, models
@@ -6,7 +6,7 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
- ('ishtar_common', '0230_auto_20231018_1551'),
+ ('ishtar_common', '0231_default_mandatory_keys'),
('archaeological_warehouse', '0118_auto_20230807_1106'),
]
diff --git a/ishtar_common/migrations/0230_auto_20231018_1551.py b/ishtar_common/migrations/0230_auto_20231024_1045.py
index c2fc2abe0..f9715bd65 100644
--- a/ishtar_common/migrations/0230_auto_20231018_1551.py
+++ b/ishtar_common/migrations/0230_auto_20231024_1045.py
@@ -1,4 +1,4 @@
-# Generated by Django 2.2.24 on 2023-10-18 15:51
+# Generated by Django 2.2.24 on 2023-10-24 10:45
import django.core.validators
from django.db import migrations, models
@@ -153,6 +153,21 @@ class Migration(migrations.Migration):
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='imports', to='ishtar_common.ImportGroup', verbose_name='Group'),
),
migrations.CreateModel(
+ name='ImportLineError',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('line', models.PositiveIntegerField(verbose_name='Line')),
+ ('ignored', models.BooleanField(default=False, verbose_name='Ignored')),
+ ('import_item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='error_lines', to='ishtar_common.Import')),
+ ],
+ options={
+ 'verbose_name': 'Import - Ignored error',
+ 'verbose_name_plural': 'Import - Ignored error',
+ 'ordering': ('import_item', 'line'),
+ 'unique_together': {('line', 'import_item')},
+ },
+ ),
+ migrations.CreateModel(
name='ImporterGroupImporter',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
diff --git a/ishtar_common/migrations/0231_default_mandatory_keys.py b/ishtar_common/migrations/0231_default_mandatory_keys.py
index 568e60b9a..cd245322a 100644
--- a/ishtar_common/migrations/0231_default_mandatory_keys.py
+++ b/ishtar_common/migrations/0231_default_mandatory_keys.py
@@ -30,7 +30,7 @@ def migrate(apps, __):
class Migration(migrations.Migration):
dependencies = [
- ('ishtar_common', '0230_auto_20231018_1551'),
+ ('ishtar_common', '0230_auto_20231024_1045'),
]
operations = [
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index a38fc3c3e..d7a546696 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -107,6 +107,7 @@ from ishtar_common.model_managers import (
)
from ishtar_common.model_merging import merge_model_objects
from ishtar_common.models_imports import (
+ ImportLineError,
ImporterModel,
ImporterType,
ImporterGroup,
@@ -181,6 +182,7 @@ __all__ = [
"ImporterType",
"ImporterGroup",
"ImportGroup",
+ "ImportLineError",
"ImporterGroupImporter",
"ImporterDefault",
"ImporterDefaultValues",
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index a240f4326..77dad558e 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -47,6 +47,7 @@ from django.core.exceptions import ValidationError, SuspiciousOperation
from django.core.files import File
from django.core.files.base import ContentFile
from django.core.validators import validate_comma_separated_integer_list, MinValueValidator
+from django.db.models import Q
from django.db.models.base import ModelBase
from django.db.models.signals import pre_delete
from django.template.defaultfilters import slugify
@@ -1423,7 +1424,7 @@ class BaseImport(models.Model, OwnPerms, SheetItem):
return q
IshtarUser = apps.get_model("ishtar_common", "IshtarUser")
ishtar_user = IshtarUser.objects.get(pk=user.pk)
- q = q.filter(user=ishtar_user)
+ q = q.filter(Q(user=ishtar_user) | Q(importer_type__users__pk=ishtar_user.pk))
return q
@classmethod
@@ -1924,9 +1925,8 @@ class Import(BaseImport):
return "{} | {}".format(self.name or "-", self.importer_type)
def __init__(self, *args, **kwargs):
- returned = super().__init__(*args, **kwargs)
+ super().__init__(*args, **kwargs)
self._initial_imported_file = self.imported_file.path if self.imported_file else ""
- return returned
def is_available(self, ishtar_user) -> bool:
if ishtar_user.is_superuser or self.user == ishtar_user:
@@ -1981,7 +1981,11 @@ class Import(BaseImport):
@property
def has_error(self) -> bool:
- return bool(self.error_file)
+ if not self.error_file:
+ return False
+ if self.error_file and not self.error_lines.count():
+ self.parse_error_file()
+ return bool(self.error_lines.filter(ignored=False).count())
@property
def pre_import_form_is_valid(self) -> bool:
@@ -2686,6 +2690,17 @@ class Import(BaseImport):
def get_all_updated(self):
return self._get_all_related("import_updated_")
+ def parse_error_file(self):
+ if not self.error_file or not self.error_file.path:
+ ImportLineError.objects.filter(import_item=self).delete()
+ return
+ with open(self.error_file.path, "r") as error_file:
+ reader = csv.reader(error_file)
+ for idx, line in enumerate(reader):
+ if not idx: # header
+ continue
+ ImportLineError.objects.get_or_create(import_item=self, line=idx)
+
def save(self, *args, **kwargs):
if self.imported_file:
if self._initial_imported_file != self.imported_file.path or not self.imported_values:
@@ -2693,6 +2708,8 @@ class Import(BaseImport):
elif self.imported_values:
self.imported_values = None
super().save(*args, **kwargs)
+ if not getattr(self, "_no_parse_error_file", False):
+ self.parse_error_file()
def pre_delete_import(sender, **kwargs):
@@ -2723,6 +2740,39 @@ def pre_delete_import(sender, **kwargs):
pre_delete.connect(pre_delete_import, sender=Import)
+class ImportLineError(models.Model):
+ import_item = models.ForeignKey(Import, on_delete=models.CASCADE, related_name="error_lines")
+ line = models.PositiveIntegerField(_("Line"))
+ ignored = models.BooleanField(_("Ignored"), default=False)
+
+ class Meta:
+ verbose_name = _("Import - Ignored error")
+ verbose_name_plural = _("Import - Ignored error")
+ unique_together = ("line", "import_item")
+ ordering = ("import_item", "line")
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._initial_ignored = self.ignored
+
+ def save(self, *args, **kwargs):
+ super().save(*args, **kwargs)
+ # reevaluate error status of the import
+ if self._initial_ignored == self.ignored:
+ return
+ has_non_ignored_errors = self.import_item.error_lines.filter(ignored=False).count()
+ modified = False
+ if has_non_ignored_errors and self.import_item.state == "F":
+ self.import_item.state = "FE"
+ modified = True
+ elif not has_non_ignored_errors and self.import_item.state == "FE":
+ self.import_item.state = "F"
+ modified = True
+ if modified:
+ self.import_item._no_parse_error_file = True
+ self.import_item.save()
+
+
class ImportColumnValue(models.Model):
"""
Value in a column for pre-import columns
diff --git a/ishtar_common/static/js/ishtar.js b/ishtar_common/static/js/ishtar.js
index 9797c7d4a..10ca7bd6c 100644
--- a/ishtar_common/static/js/ishtar.js
+++ b/ishtar_common/static/js/ishtar.js
@@ -124,8 +124,9 @@ var activate_all_search_url = '/activate-all-search/';
var activate_own_search_url = '/activate-own-search/';
var activate_advanced_url = '/activate-advanced-menu/';
var activate_simple_url = '/activate-simple-menu/';
-var shortcut_menu_hide_url = '/hide-shortcut-menu/'
-var shortcut_menu_show_url = '/show-shortcut-menu/'
+var shortcut_menu_hide_url = '/hide-shortcut-menu/';
+var shortcut_menu_show_url = '/show-shortcut-menu/';
+var ignore_error_line_url = "/import-ignore-line/";
function init_shortcut_menu(html){
close_wait();
@@ -2163,5 +2164,23 @@ var import_table_refresh_import_list = function(data){
item["current_line"] + "/" + item["number_of_line"]
);
}
+ if (item["has_error"]){
+ $("#import-" + item["full_id"]).addClass("import-row-error");
+ } else {
+ $("#import-" + item["full_id"]).removeClass("import-row-error");
+ }
}
}
+
+var import_csv_check_ignored = function(){
+ let tr = $(this).parent().parent();
+ if (this.checked){
+ tr.addClass("import-line-ignored");
+ } else {
+ tr.removeClass("import-line-ignored");
+ }
+ $.get(url_path + ignore_error_line_url + this.value + "/", function(){
+ let import_ids = [tr.parent().parent().attr("data-import")];
+ import_table_update_import_list(import_ids);
+ });
+}
diff --git a/ishtar_common/templates/ishtar/blocks/view_import_csv.html b/ishtar_common/templates/ishtar/blocks/view_import_csv.html
index f1b089ce2..ba9f43d50 100644
--- a/ishtar_common/templates/ishtar/blocks/view_import_csv.html
+++ b/ishtar_common/templates/ishtar/blocks/view_import_csv.html
@@ -1,4 +1,4 @@
-{% load i18n %}
+{% load i18n l10n %}
<div class="modal-dialog full modal-lg">
<div class="modal-content">
@@ -12,21 +12,27 @@
</button>
</div>
<div class="table-scroll">
- <table class="table table-striped table-bordered">
+ <table class="table table-striped table-bordered" data-import="{{import_id}}">
<thead>
<tr>
<th>&nbsp;</th>
+ {% if has_line_errors %}<th>{% trans "Ignore" %}</th>{% endif %}
{% for head in header %}<th>{{head}}</th>{% endfor %}
</tr>
</thead>
<tbody>
{% for line in content %}
- <tr>
+ <tr{% if has_line_errors and line.0.1 %} class="import-line-ignored"{% endif%}>
<td>{{ forloop.counter }}</td>
- {% for cell in line %}<td>{{cell}}</td>{% endfor %}
+ {% for cell in line %}<td>
+ {% if has_line_errors and not forloop.counter0 %}<input class="check-ignored" value="{{cell.0}}"{% if cell.1%} checked="checked"{% endif %} type="checkbox" />{% else %}{{cell}}{% endif %}
+ </td>{% endfor %}
</tr>{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
+<script type="text/javascript">{% localize off %}
+$(".check-ignored").click(import_csv_check_ignored);
+{% endlocalize %}</script> \ No newline at end of file
diff --git a/ishtar_common/urls.py b/ishtar_common/urls.py
index 9a5ee4f1c..52e1324b2 100644
--- a/ishtar_common/urls.py
+++ b/ishtar_common/urls.py
@@ -284,6 +284,11 @@ urlpatterns = [
check_rights(["change_import"])(views.ImportPreFormView.as_view()),
name="import_pre_import_form",
),
+ url(
+ r"^import-ignore-line/(?P<line_id>[0-9]+)/$",
+ views.line_error,
+ name="import_ignore_line",
+ ),
url(r"^profile(?:/(?P<pk>[0-9]+))?/$", views.ProfileEdit.as_view(), name="profile"),
url(
r"^save-search/(?P<app_label>[a-z-]+)/(?P<model>[a-z-]+)/$",
diff --git a/ishtar_common/views.py b/ishtar_common/views.py
index 8f8ecb5a6..b91f3202c 100644
--- a/ishtar_common/views.py
+++ b/ishtar_common/views.py
@@ -2075,6 +2075,7 @@ def import_get_status(request, current_right=None):
"full_id": ("group-" if key == "group" else "") + str(item.id),
"state": item.state,
"status": str(item.status),
+ "has_error": item.has_error
}
if key == "import":
item_dct.update({
@@ -2182,6 +2183,16 @@ class ImportCSVView(IshtarMixin, LoginRequiredMixin, TemplateView):
encoding = self.import_item.encoding
data["icon"], data["target"] = self.TITLES[self.kwargs["target"]]
data["title"] = str(self.import_item)
+ line_errors = []
+ has_line_errors = False
+ if self.kwargs["target"] == "error":
+ has_line_errors = True
+ q = models.ImportLineError.objects.filter(import_item=self.import_item)
+ if not q.count():
+ self.import_item.parse_error_file()
+ line_errors = models.ImportLineError.objects.filter(
+ import_item=self.import_item).values_list("pk", "ignored")
+ data["has_line_errors"] = has_line_errors
data["content"] = []
with open(self.csv_file.path, "r", encoding=encoding) as f:
reader = csv.reader(f)
@@ -2189,11 +2200,33 @@ class ImportCSVView(IshtarMixin, LoginRequiredMixin, TemplateView):
if not idx:
data["header"] = line
continue
+ if line_errors and len(line_errors) >= idx:
+ line = [line_errors[idx - 1]] + line
data["content"].append(line)
data["window_id"] = "csv-view-" + (self.kwargs.get("group", "") or "") + str(self.import_item.pk)
+ data["import_id"] = self.import_item.pk
return data
+def line_error(request, line_id):
+ """
+ Set or unset ignored state of a csv error file
+ """
+ user = request.user
+ if not user.pk:
+ raise Http404()
+ q = models.ImportLineError.objects.filter(pk=line_id)
+ if not q.count():
+ return
+ line = q.all()[0]
+ q = models.Import.query_can_access(request.user).filter(pk=line.import_item_id)
+ if not q.count():
+ raise Http404()
+ line.ignored = not line.ignored
+ line.save()
+ return HttpResponse(content_type="text/plain")
+
+
class PersonCreate(LoginRequiredMixin, CreateView):
model = models.Person
form_class = forms.BasePersonForm
diff --git a/scss/custom.scss b/scss/custom.scss
index 3b6d803b3..749d2becf 100644
--- a/scss/custom.scss
+++ b/scss/custom.scss
@@ -304,6 +304,10 @@ pre {
background-color: lighten(red, 40%);
}
+.import-line-ignored {
+ color: $gray-500;
+}
+
#import-container {
overflow: scroll;