From 94a4a57148d2c81dbbc1251b7e79ba09282af217 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Mon, 23 Oct 2023 12:36:44 +0200 Subject: ✨ imports: allow errors to be pointed out line by line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../0113_contextrecord_imports_updated.py | 4 +- .../migrations/0112_file_imports_updated.py | 4 +- .../migrations/0113_auto_20231018_1553.py | 39 ---- .../migrations/0113_auto_20231024_1050.py | 39 ++++ .../migrations/0113_auto_20231018_1551.py | 39 ---- .../migrations/0113_auto_20231024_1045.py | 39 ++++ .../migrations/0119_auto_20231018_1553.py | 24 --- .../migrations/0119_auto_20231024_1050.py | 24 +++ .../migrations/0230_auto_20231018_1551.py | 183 ------------------- .../migrations/0230_auto_20231024_1045.py | 198 +++++++++++++++++++++ .../migrations/0231_default_mandatory_keys.py | 2 +- ishtar_common/models.py | 2 + ishtar_common/models_imports.py | 58 +++++- ishtar_common/static/js/ishtar.js | 23 ++- .../templates/ishtar/blocks/view_import_csv.html | 14 +- ishtar_common/urls.py | 5 + ishtar_common/views.py | 33 ++++ scss/custom.scss | 4 + 18 files changed, 434 insertions(+), 300 deletions(-) delete mode 100644 archaeological_finds/migrations/0113_auto_20231018_1553.py create mode 100644 archaeological_finds/migrations/0113_auto_20231024_1050.py delete mode 100644 archaeological_operations/migrations/0113_auto_20231018_1551.py create mode 100644 archaeological_operations/migrations/0113_auto_20231024_1045.py delete mode 100644 archaeological_warehouse/migrations/0119_auto_20231018_1553.py create mode 100644 archaeological_warehouse/migrations/0119_auto_20231024_1050.py delete mode 100644 ishtar_common/migrations/0230_auto_20231018_1551.py create mode 100644 ishtar_common/migrations/0230_auto_20231024_1045.py diff --git a/archaeological_context_records/migrations/0113_contextrecord_imports_updated.py b/archaeological_context_records/migrations/0113_contextrecord_imports_updated.py index 6c9a5e085..b68f81937 100644 --- a/archaeological_context_records/migrations/0113_contextrecord_imports_updated.py +++ b/archaeological_context_records/migrations/0113_contextrecord_imports_updated.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2.24 on 2023-10-18 15:51 +# Generated by Django 2.2.24 on 2023-10-24 10:45 from django.db import migrations, models @@ -6,7 +6,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('ishtar_common', '0230_auto_20231018_1551'), + ('ishtar_common', '0230_auto_20231024_1045'), ('archaeological_context_records', '0112_migrate_created'), ] diff --git a/archaeological_files/migrations/0112_file_imports_updated.py b/archaeological_files/migrations/0112_file_imports_updated.py index d227e0487..ab076fbc8 100644 --- a/archaeological_files/migrations/0112_file_imports_updated.py +++ b/archaeological_files/migrations/0112_file_imports_updated.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2.24 on 2023-10-18 15:53 +# Generated by Django 2.2.24 on 2023-10-24 10:50 from django.db import migrations, models @@ -6,7 +6,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('ishtar_common', '0230_auto_20231018_1551'), + ('ishtar_common', '0231_default_mandatory_keys'), ('archaeological_files', '0111_migrate_created'), ] diff --git a/archaeological_finds/migrations/0113_auto_20231018_1553.py b/archaeological_finds/migrations/0113_auto_20231018_1553.py deleted file mode 100644 index e9bfad7fd..000000000 --- a/archaeological_finds/migrations/0113_auto_20231018_1553.py +++ /dev/null @@ -1,39 +0,0 @@ -# Generated by Django 2.2.24 on 2023-10-18 15:53 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ishtar_common', '0230_auto_20231018_1551'), - ('archaeological_finds', '0112_migrate_created'), - ] - - operations = [ - migrations.AddField( - model_name='basefind', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_finds_basefind', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='find', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_finds_find', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='property', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_finds_property', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='treatment', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_finds_treatment', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='treatmentfile', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_finds_treatmentfile', to='ishtar_common.Import'), - ), - ] diff --git a/archaeological_finds/migrations/0113_auto_20231024_1050.py b/archaeological_finds/migrations/0113_auto_20231024_1050.py new file mode 100644 index 000000000..a1ede77e6 --- /dev/null +++ b/archaeological_finds/migrations/0113_auto_20231024_1050.py @@ -0,0 +1,39 @@ +# Generated by Django 2.2.24 on 2023-10-24 10:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0231_default_mandatory_keys'), + ('archaeological_finds', '0112_migrate_created'), + ] + + operations = [ + migrations.AddField( + model_name='basefind', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_finds_basefind', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='find', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_finds_find', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='property', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_finds_property', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='treatment', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_finds_treatment', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='treatmentfile', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_finds_treatmentfile', to='ishtar_common.Import'), + ), + ] diff --git a/archaeological_operations/migrations/0113_auto_20231018_1551.py b/archaeological_operations/migrations/0113_auto_20231018_1551.py deleted file mode 100644 index 11d1143d0..000000000 --- a/archaeological_operations/migrations/0113_auto_20231018_1551.py +++ /dev/null @@ -1,39 +0,0 @@ -# Generated by Django 2.2.24 on 2023-10-18 15:51 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ishtar_common', '0230_auto_20231018_1551'), - ('archaeological_operations', '0112_migrate_created'), - ] - - operations = [ - migrations.AddField( - model_name='administrativeact', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_operations_administrativeact', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='archaeologicalsite', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_operations_archaeologicalsite', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='operation', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_operations_operation', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='parcel', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_operations_parcel', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='parcelowner', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_operations_parcelowner', to='ishtar_common.Import'), - ), - ] diff --git a/archaeological_operations/migrations/0113_auto_20231024_1045.py b/archaeological_operations/migrations/0113_auto_20231024_1045.py new file mode 100644 index 000000000..bb0b8ac0e --- /dev/null +++ b/archaeological_operations/migrations/0113_auto_20231024_1045.py @@ -0,0 +1,39 @@ +# Generated by Django 2.2.24 on 2023-10-24 10:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0230_auto_20231024_1045'), + ('archaeological_operations', '0112_migrate_created'), + ] + + operations = [ + migrations.AddField( + model_name='administrativeact', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_operations_administrativeact', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='archaeologicalsite', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_operations_archaeologicalsite', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='operation', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_operations_operation', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='parcel', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_operations_parcel', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='parcelowner', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_operations_parcelowner', to='ishtar_common.Import'), + ), + ] diff --git a/archaeological_warehouse/migrations/0119_auto_20231018_1553.py b/archaeological_warehouse/migrations/0119_auto_20231018_1553.py deleted file mode 100644 index b84564711..000000000 --- a/archaeological_warehouse/migrations/0119_auto_20231018_1553.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 2.2.24 on 2023-10-18 15:53 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ishtar_common', '0230_auto_20231018_1551'), - ('archaeological_warehouse', '0118_auto_20230807_1106'), - ] - - operations = [ - migrations.AddField( - model_name='container', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_warehouse_container', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='warehouse', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_warehouse_warehouse', to='ishtar_common.Import'), - ), - ] diff --git a/archaeological_warehouse/migrations/0119_auto_20231024_1050.py b/archaeological_warehouse/migrations/0119_auto_20231024_1050.py new file mode 100644 index 000000000..5b908736d --- /dev/null +++ b/archaeological_warehouse/migrations/0119_auto_20231024_1050.py @@ -0,0 +1,24 @@ +# Generated by Django 2.2.24 on 2023-10-24 10:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0231_default_mandatory_keys'), + ('archaeological_warehouse', '0118_auto_20230807_1106'), + ] + + operations = [ + migrations.AddField( + model_name='container', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_warehouse_container', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='warehouse', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_warehouse_warehouse', to='ishtar_common.Import'), + ), + ] diff --git a/ishtar_common/migrations/0230_auto_20231018_1551.py b/ishtar_common/migrations/0230_auto_20231018_1551.py deleted file mode 100644 index c2fc2abe0..000000000 --- a/ishtar_common/migrations/0230_auto_20231018_1551.py +++ /dev/null @@ -1,183 +0,0 @@ -# Generated by Django 2.2.24 on 2023-10-18 15:51 - -import django.core.validators -from django.db import migrations, models -import django.db.models.deletion -import ishtar_common.utils - - -class Migration(migrations.Migration): - - dependencies = [ - ('ishtar_common', '0229_auto_20230608_1303'), - ] - - operations = [ - migrations.CreateModel( - name='ImporterGroup', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=200, verbose_name='Name')), - ('slug', models.SlugField(max_length=100, unique=True, verbose_name='Slug')), - ('description', models.TextField(blank=True, default='', verbose_name='Description')), - ('available', models.BooleanField(default=True, verbose_name='Available')), - ('users', models.ManyToManyField(blank=True, to='ishtar_common.IshtarUser', verbose_name='Users')), - ], - options={ - 'verbose_name': 'Importer - Group', - 'verbose_name_plural': 'Importer - Groups', - 'ordering': ('name',), - }, - ), - migrations.AlterModelOptions( - name='import', - options={'permissions': (('view_own_import', 'Can view own Import'), ('add_own_import', 'Can add own Import'), ('change_own_import', 'Can change own Import'), ('delete_own_import', 'Can delete own Import')), 'verbose_name': 'Import - Import', 'verbose_name_plural': 'Import - Imports'}, - ), - migrations.AlterModelOptions( - name='itemkey', - options={'verbose_name': 'Import - Item key', 'verbose_name_plural': 'Import - Item keys'}, - ), - migrations.AddField( - model_name='document', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_document', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='geovectordata', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_geovectordata', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='import', - name='imported_values', - field=models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Imported values'), - ), - migrations.AddField( - model_name='import', - name='next_import', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='imports', to='ishtar_common.Import', verbose_name='Next import'), - ), - migrations.AddField( - model_name='importerdefault', - name='required_fields', - field=models.CharField(blank=True, default='', help_text='Theses defaults values only apply if the designated fields are not empty. Use the "__" notation to pass between models.Leave empty to always apply.', max_length=500, verbose_name='Required fields'), - ), - migrations.AddField( - model_name='importertype', - name='archive_required', - field=models.BooleanField(default=False, verbose_name='Archive required'), - ), - migrations.AddField( - model_name='importertype', - name='ignore_errors', - field=models.TextField(blank=True, help_text='If an error is encountered with the following character strings, the error is not reported in the error file. Each message is separated with a line break.', null=True, verbose_name='Error messages to ignore'), - ), - migrations.AddField( - model_name='importertype', - name='is_import', - field=models.BooleanField(default=True, verbose_name='Can be import'), - ), - migrations.AddField( - model_name='importertype', - name='pre_import_message', - field=models.TextField(blank=True, default='', max_length=500, verbose_name='Pre-import form message'), - ), - migrations.AddField( - model_name='importertype', - name='tab_number', - field=models.PositiveIntegerField(default=1, help_text='When using an Excel or Calc file choose the tab number. Keep it to 1 by default.', validators=[django.core.validators.MinValueValidator(1)], verbose_name='Tab number'), - ), - migrations.AddField( - model_name='organization', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_organization', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='person', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_person', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='town', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_town', to='ishtar_common.Import'), - ), - migrations.AlterField( - model_name='import', - name='imported_images', - field=models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Associated documents (zip file)'), - ), - migrations.AlterField( - model_name='importercolumn', - name='col_number', - field=models.SmallIntegerField(default=1, help_text='Column number in the table. Put 0 or negative number for pre-importer field.', verbose_name='Column number'), - ), - migrations.AlterField( - model_name='importerdefault', - name='target', - field=models.CharField(help_text='The target of the default values. Can be set to empty with "-". Use the "__" notation to pass between models.', max_length=500, verbose_name='Target'), - ), - migrations.AlterField( - model_name='ishtarsiteprofile', - name='account_naming_style', - field=models.CharField(choices=[('NF', 'name.firstname'), ('FN', 'firstname.name')], default='FN', max_length=2, verbose_name='Naming style for accounts'), - ), - migrations.CreateModel( - name='ImportGroup', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=500, null=True, verbose_name='Name')), - ('imported_file', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Imported file')), - ('imported_images', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Associated documents (zip file)')), - ('archive_file', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=255, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Archive file')), - ('encoding', models.CharField(choices=[('windows-1252', 'windows-1252'), ('ISO-8859-15', 'ISO-8859-15'), ('utf-8', 'utf-8')], default='utf-8', help_text='Only required for CSV file', max_length=15, verbose_name='Encoding')), - ('csv_sep', models.CharField(choices=[(',', ','), (';', ';'), ('|', '|')], default=',', help_text='Separator for CSV file. Standard is comma but Microsoft Excel do not follow this standard and use semi-colon.', max_length=1, verbose_name='CSV separator')), - ('skip_lines', models.IntegerField(default=1, help_text='Number of header lines in your file (can be 0 and should be 0 for geopackage or Shapefile).', verbose_name='Skip lines')), - ('creation_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Creation date')), - ('end_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='End date')), - ('current_import', models.PositiveIntegerField(blank=True, null=True, verbose_name='Current import')), - ('state', models.CharField(choices=[('C', 'Created'), ('AP', 'Analyse in progress'), ('A', 'Analysed'), ('IQ', 'Import in queue'), ('IP', 'Import in progress'), ('PP', 'Post-processing in progress'), ('FE', 'Finished with errors'), ('F', 'Finished'), ('AC', 'Archived')], default='C', max_length=2, verbose_name='State')), - ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ImporterGroup', verbose_name='Importer group type')), - ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.IshtarUser')), - ], - options={ - 'verbose_name': 'Import - Group', - 'verbose_name_plural': 'Import - Groups', - 'permissions': (('view_own_importgroup', 'Can view own Import Group'), ('add_own_importgroup', 'Can add own Import Group'), ('change_own_importgroup', 'Can change own Import Group'), ('delete_own_importgroup', 'Can delete own Import Group')), - }, - bases=(models.Model, ishtar_common.utils.OwnPerms, ishtar_common.utils.SheetItem), - ), - migrations.AddField( - model_name='import', - name='group', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='imports', to='ishtar_common.ImportGroup', verbose_name='Group'), - ), - migrations.CreateModel( - name='ImporterGroupImporter', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('order', models.PositiveIntegerField(default=10, validators=[django.core.validators.MinValueValidator(1)], verbose_name='Order')), - ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='importer_types', to='ishtar_common.ImporterGroup')), - ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='ishtar_common.ImporterType')), - ], - options={ - 'verbose_name': 'Importer - Group <-> Importer', - 'ordering': ('group', 'order'), - 'unique_together': {('group', 'order')}, - }, - ), - migrations.CreateModel( - name='ImportColumnValue', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('value', models.TextField(blank=True, default='')), - ('column', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ImporterColumn')), - ('import_item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.Import')), - ], - options={ - 'verbose_name': 'Import - Pre-import value', - 'verbose_name_plural': 'Import - Pre-import values', - 'unique_together': {('column', 'import_item')}, - }, - ), - ] diff --git a/ishtar_common/migrations/0230_auto_20231024_1045.py b/ishtar_common/migrations/0230_auto_20231024_1045.py new file mode 100644 index 000000000..f9715bd65 --- /dev/null +++ b/ishtar_common/migrations/0230_auto_20231024_1045.py @@ -0,0 +1,198 @@ +# Generated by Django 2.2.24 on 2023-10-24 10:45 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import ishtar_common.utils + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0229_auto_20230608_1303'), + ] + + operations = [ + migrations.CreateModel( + name='ImporterGroup', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=200, verbose_name='Name')), + ('slug', models.SlugField(max_length=100, unique=True, verbose_name='Slug')), + ('description', models.TextField(blank=True, default='', verbose_name='Description')), + ('available', models.BooleanField(default=True, verbose_name='Available')), + ('users', models.ManyToManyField(blank=True, to='ishtar_common.IshtarUser', verbose_name='Users')), + ], + options={ + 'verbose_name': 'Importer - Group', + 'verbose_name_plural': 'Importer - Groups', + 'ordering': ('name',), + }, + ), + migrations.AlterModelOptions( + name='import', + options={'permissions': (('view_own_import', 'Can view own Import'), ('add_own_import', 'Can add own Import'), ('change_own_import', 'Can change own Import'), ('delete_own_import', 'Can delete own Import')), 'verbose_name': 'Import - Import', 'verbose_name_plural': 'Import - Imports'}, + ), + migrations.AlterModelOptions( + name='itemkey', + options={'verbose_name': 'Import - Item key', 'verbose_name_plural': 'Import - Item keys'}, + ), + migrations.AddField( + model_name='document', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_document', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='geovectordata', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_geovectordata', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='import', + name='imported_values', + field=models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Imported values'), + ), + migrations.AddField( + model_name='import', + name='next_import', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='imports', to='ishtar_common.Import', verbose_name='Next import'), + ), + migrations.AddField( + model_name='importerdefault', + name='required_fields', + field=models.CharField(blank=True, default='', help_text='Theses defaults values only apply if the designated fields are not empty. Use the "__" notation to pass between models.Leave empty to always apply.', max_length=500, verbose_name='Required fields'), + ), + migrations.AddField( + model_name='importertype', + name='archive_required', + field=models.BooleanField(default=False, verbose_name='Archive required'), + ), + migrations.AddField( + model_name='importertype', + name='ignore_errors', + field=models.TextField(blank=True, help_text='If an error is encountered with the following character strings, the error is not reported in the error file. Each message is separated with a line break.', null=True, verbose_name='Error messages to ignore'), + ), + migrations.AddField( + model_name='importertype', + name='is_import', + field=models.BooleanField(default=True, verbose_name='Can be import'), + ), + migrations.AddField( + model_name='importertype', + name='pre_import_message', + field=models.TextField(blank=True, default='', max_length=500, verbose_name='Pre-import form message'), + ), + migrations.AddField( + model_name='importertype', + name='tab_number', + field=models.PositiveIntegerField(default=1, help_text='When using an Excel or Calc file choose the tab number. Keep it to 1 by default.', validators=[django.core.validators.MinValueValidator(1)], verbose_name='Tab number'), + ), + migrations.AddField( + model_name='organization', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_organization', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='person', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_person', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='town', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_town', to='ishtar_common.Import'), + ), + migrations.AlterField( + model_name='import', + name='imported_images', + field=models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Associated documents (zip file)'), + ), + migrations.AlterField( + model_name='importercolumn', + name='col_number', + field=models.SmallIntegerField(default=1, help_text='Column number in the table. Put 0 or negative number for pre-importer field.', verbose_name='Column number'), + ), + migrations.AlterField( + model_name='importerdefault', + name='target', + field=models.CharField(help_text='The target of the default values. Can be set to empty with "-". Use the "__" notation to pass between models.', max_length=500, verbose_name='Target'), + ), + migrations.AlterField( + model_name='ishtarsiteprofile', + name='account_naming_style', + field=models.CharField(choices=[('NF', 'name.firstname'), ('FN', 'firstname.name')], default='FN', max_length=2, verbose_name='Naming style for accounts'), + ), + migrations.CreateModel( + name='ImportGroup', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=500, null=True, verbose_name='Name')), + ('imported_file', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Imported file')), + ('imported_images', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Associated documents (zip file)')), + ('archive_file', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=255, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Archive file')), + ('encoding', models.CharField(choices=[('windows-1252', 'windows-1252'), ('ISO-8859-15', 'ISO-8859-15'), ('utf-8', 'utf-8')], default='utf-8', help_text='Only required for CSV file', max_length=15, verbose_name='Encoding')), + ('csv_sep', models.CharField(choices=[(',', ','), (';', ';'), ('|', '|')], default=',', help_text='Separator for CSV file. Standard is comma but Microsoft Excel do not follow this standard and use semi-colon.', max_length=1, verbose_name='CSV separator')), + ('skip_lines', models.IntegerField(default=1, help_text='Number of header lines in your file (can be 0 and should be 0 for geopackage or Shapefile).', verbose_name='Skip lines')), + ('creation_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Creation date')), + ('end_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='End date')), + ('current_import', models.PositiveIntegerField(blank=True, null=True, verbose_name='Current import')), + ('state', models.CharField(choices=[('C', 'Created'), ('AP', 'Analyse in progress'), ('A', 'Analysed'), ('IQ', 'Import in queue'), ('IP', 'Import in progress'), ('PP', 'Post-processing in progress'), ('FE', 'Finished with errors'), ('F', 'Finished'), ('AC', 'Archived')], default='C', max_length=2, verbose_name='State')), + ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ImporterGroup', verbose_name='Importer group type')), + ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.IshtarUser')), + ], + options={ + 'verbose_name': 'Import - Group', + 'verbose_name_plural': 'Import - Groups', + 'permissions': (('view_own_importgroup', 'Can view own Import Group'), ('add_own_importgroup', 'Can add own Import Group'), ('change_own_importgroup', 'Can change own Import Group'), ('delete_own_importgroup', 'Can delete own Import Group')), + }, + bases=(models.Model, ishtar_common.utils.OwnPerms, ishtar_common.utils.SheetItem), + ), + migrations.AddField( + model_name='import', + name='group', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='imports', to='ishtar_common.ImportGroup', verbose_name='Group'), + ), + migrations.CreateModel( + name='ImportLineError', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('line', models.PositiveIntegerField(verbose_name='Line')), + ('ignored', models.BooleanField(default=False, verbose_name='Ignored')), + ('import_item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='error_lines', to='ishtar_common.Import')), + ], + options={ + 'verbose_name': 'Import - Ignored error', + 'verbose_name_plural': 'Import - Ignored error', + 'ordering': ('import_item', 'line'), + 'unique_together': {('line', 'import_item')}, + }, + ), + migrations.CreateModel( + name='ImporterGroupImporter', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('order', models.PositiveIntegerField(default=10, validators=[django.core.validators.MinValueValidator(1)], verbose_name='Order')), + ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='importer_types', to='ishtar_common.ImporterGroup')), + ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='ishtar_common.ImporterType')), + ], + options={ + 'verbose_name': 'Importer - Group <-> Importer', + 'ordering': ('group', 'order'), + 'unique_together': {('group', 'order')}, + }, + ), + migrations.CreateModel( + name='ImportColumnValue', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('value', models.TextField(blank=True, default='')), + ('column', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ImporterColumn')), + ('import_item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.Import')), + ], + options={ + 'verbose_name': 'Import - Pre-import value', + 'verbose_name_plural': 'Import - Pre-import values', + 'unique_together': {('column', 'import_item')}, + }, + ), + ] diff --git a/ishtar_common/migrations/0231_default_mandatory_keys.py b/ishtar_common/migrations/0231_default_mandatory_keys.py index 568e60b9a..cd245322a 100644 --- a/ishtar_common/migrations/0231_default_mandatory_keys.py +++ b/ishtar_common/migrations/0231_default_mandatory_keys.py @@ -30,7 +30,7 @@ def migrate(apps, __): class Migration(migrations.Migration): dependencies = [ - ('ishtar_common', '0230_auto_20231018_1551'), + ('ishtar_common', '0230_auto_20231024_1045'), ] operations = [ diff --git a/ishtar_common/models.py b/ishtar_common/models.py index a38fc3c3e..d7a546696 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -107,6 +107,7 @@ from ishtar_common.model_managers import ( ) from ishtar_common.model_merging import merge_model_objects from ishtar_common.models_imports import ( + ImportLineError, ImporterModel, ImporterType, ImporterGroup, @@ -181,6 +182,7 @@ __all__ = [ "ImporterType", "ImporterGroup", "ImportGroup", + "ImportLineError", "ImporterGroupImporter", "ImporterDefault", "ImporterDefaultValues", diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py index a240f4326..77dad558e 100644 --- a/ishtar_common/models_imports.py +++ b/ishtar_common/models_imports.py @@ -47,6 +47,7 @@ from django.core.exceptions import ValidationError, SuspiciousOperation from django.core.files import File from django.core.files.base import ContentFile from django.core.validators import validate_comma_separated_integer_list, MinValueValidator +from django.db.models import Q from django.db.models.base import ModelBase from django.db.models.signals import pre_delete from django.template.defaultfilters import slugify @@ -1423,7 +1424,7 @@ class BaseImport(models.Model, OwnPerms, SheetItem): return q IshtarUser = apps.get_model("ishtar_common", "IshtarUser") ishtar_user = IshtarUser.objects.get(pk=user.pk) - q = q.filter(user=ishtar_user) + q = q.filter(Q(user=ishtar_user) | Q(importer_type__users__pk=ishtar_user.pk)) return q @classmethod @@ -1924,9 +1925,8 @@ class Import(BaseImport): return "{} | {}".format(self.name or "-", self.importer_type) def __init__(self, *args, **kwargs): - returned = super().__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._initial_imported_file = self.imported_file.path if self.imported_file else "" - return returned def is_available(self, ishtar_user) -> bool: if ishtar_user.is_superuser or self.user == ishtar_user: @@ -1981,7 +1981,11 @@ class Import(BaseImport): @property def has_error(self) -> bool: - return bool(self.error_file) + if not self.error_file: + return False + if self.error_file and not self.error_lines.count(): + self.parse_error_file() + return bool(self.error_lines.filter(ignored=False).count()) @property def pre_import_form_is_valid(self) -> bool: @@ -2686,6 +2690,17 @@ class Import(BaseImport): def get_all_updated(self): return self._get_all_related("import_updated_") + def parse_error_file(self): + if not self.error_file or not self.error_file.path: + ImportLineError.objects.filter(import_item=self).delete() + return + with open(self.error_file.path, "r") as error_file: + reader = csv.reader(error_file) + for idx, line in enumerate(reader): + if not idx: # header + continue + ImportLineError.objects.get_or_create(import_item=self, line=idx) + def save(self, *args, **kwargs): if self.imported_file: if self._initial_imported_file != self.imported_file.path or not self.imported_values: @@ -2693,6 +2708,8 @@ class Import(BaseImport): elif self.imported_values: self.imported_values = None super().save(*args, **kwargs) + if not getattr(self, "_no_parse_error_file", False): + self.parse_error_file() def pre_delete_import(sender, **kwargs): @@ -2723,6 +2740,39 @@ def pre_delete_import(sender, **kwargs): pre_delete.connect(pre_delete_import, sender=Import) +class ImportLineError(models.Model): + import_item = models.ForeignKey(Import, on_delete=models.CASCADE, related_name="error_lines") + line = models.PositiveIntegerField(_("Line")) + ignored = models.BooleanField(_("Ignored"), default=False) + + class Meta: + verbose_name = _("Import - Ignored error") + verbose_name_plural = _("Import - Ignored error") + unique_together = ("line", "import_item") + ordering = ("import_item", "line") + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._initial_ignored = self.ignored + + def save(self, *args, **kwargs): + super().save(*args, **kwargs) + # reevaluate error status of the import + if self._initial_ignored == self.ignored: + return + has_non_ignored_errors = self.import_item.error_lines.filter(ignored=False).count() + modified = False + if has_non_ignored_errors and self.import_item.state == "F": + self.import_item.state = "FE" + modified = True + elif not has_non_ignored_errors and self.import_item.state == "FE": + self.import_item.state = "F" + modified = True + if modified: + self.import_item._no_parse_error_file = True + self.import_item.save() + + class ImportColumnValue(models.Model): """ Value in a column for pre-import columns diff --git a/ishtar_common/static/js/ishtar.js b/ishtar_common/static/js/ishtar.js index 9797c7d4a..10ca7bd6c 100644 --- a/ishtar_common/static/js/ishtar.js +++ b/ishtar_common/static/js/ishtar.js @@ -124,8 +124,9 @@ var activate_all_search_url = '/activate-all-search/'; var activate_own_search_url = '/activate-own-search/'; var activate_advanced_url = '/activate-advanced-menu/'; var activate_simple_url = '/activate-simple-menu/'; -var shortcut_menu_hide_url = '/hide-shortcut-menu/' -var shortcut_menu_show_url = '/show-shortcut-menu/' +var shortcut_menu_hide_url = '/hide-shortcut-menu/'; +var shortcut_menu_show_url = '/show-shortcut-menu/'; +var ignore_error_line_url = "/import-ignore-line/"; function init_shortcut_menu(html){ close_wait(); @@ -2163,5 +2164,23 @@ var import_table_refresh_import_list = function(data){ item["current_line"] + "/" + item["number_of_line"] ); } + if (item["has_error"]){ + $("#import-" + item["full_id"]).addClass("import-row-error"); + } else { + $("#import-" + item["full_id"]).removeClass("import-row-error"); + } } } + +var import_csv_check_ignored = function(){ + let tr = $(this).parent().parent(); + if (this.checked){ + tr.addClass("import-line-ignored"); + } else { + tr.removeClass("import-line-ignored"); + } + $.get(url_path + ignore_error_line_url + this.value + "/", function(){ + let import_ids = [tr.parent().parent().attr("data-import")]; + import_table_update_import_list(import_ids); + }); +} diff --git a/ishtar_common/templates/ishtar/blocks/view_import_csv.html b/ishtar_common/templates/ishtar/blocks/view_import_csv.html index f1b089ce2..ba9f43d50 100644 --- a/ishtar_common/templates/ishtar/blocks/view_import_csv.html +++ b/ishtar_common/templates/ishtar/blocks/view_import_csv.html @@ -1,4 +1,4 @@ -{% load i18n %} +{% load i18n l10n %}
@@ -12,21 +12,27 @@
- +
+ {% if has_line_errors %}{% endif %} {% for head in header %}{% endfor %} {% for line in content %} - + - {% for cell in line %}{% endfor %} + {% for cell in line %}{% endfor %} {% endfor %}
 {% trans "Ignore" %}{{head}}
{{ forloop.counter }}{{cell}} + {% if has_line_errors and not forloop.counter0 %}{% else %}{{cell}}{% endif %} +
+ \ No newline at end of file diff --git a/ishtar_common/urls.py b/ishtar_common/urls.py index 9a5ee4f1c..52e1324b2 100644 --- a/ishtar_common/urls.py +++ b/ishtar_common/urls.py @@ -284,6 +284,11 @@ urlpatterns = [ check_rights(["change_import"])(views.ImportPreFormView.as_view()), name="import_pre_import_form", ), + url( + r"^import-ignore-line/(?P[0-9]+)/$", + views.line_error, + name="import_ignore_line", + ), url(r"^profile(?:/(?P[0-9]+))?/$", views.ProfileEdit.as_view(), name="profile"), url( r"^save-search/(?P[a-z-]+)/(?P[a-z-]+)/$", diff --git a/ishtar_common/views.py b/ishtar_common/views.py index 8f8ecb5a6..b91f3202c 100644 --- a/ishtar_common/views.py +++ b/ishtar_common/views.py @@ -2075,6 +2075,7 @@ def import_get_status(request, current_right=None): "full_id": ("group-" if key == "group" else "") + str(item.id), "state": item.state, "status": str(item.status), + "has_error": item.has_error } if key == "import": item_dct.update({ @@ -2182,6 +2183,16 @@ class ImportCSVView(IshtarMixin, LoginRequiredMixin, TemplateView): encoding = self.import_item.encoding data["icon"], data["target"] = self.TITLES[self.kwargs["target"]] data["title"] = str(self.import_item) + line_errors = [] + has_line_errors = False + if self.kwargs["target"] == "error": + has_line_errors = True + q = models.ImportLineError.objects.filter(import_item=self.import_item) + if not q.count(): + self.import_item.parse_error_file() + line_errors = models.ImportLineError.objects.filter( + import_item=self.import_item).values_list("pk", "ignored") + data["has_line_errors"] = has_line_errors data["content"] = [] with open(self.csv_file.path, "r", encoding=encoding) as f: reader = csv.reader(f) @@ -2189,11 +2200,33 @@ class ImportCSVView(IshtarMixin, LoginRequiredMixin, TemplateView): if not idx: data["header"] = line continue + if line_errors and len(line_errors) >= idx: + line = [line_errors[idx - 1]] + line data["content"].append(line) data["window_id"] = "csv-view-" + (self.kwargs.get("group", "") or "") + str(self.import_item.pk) + data["import_id"] = self.import_item.pk return data +def line_error(request, line_id): + """ + Set or unset ignored state of a csv error file + """ + user = request.user + if not user.pk: + raise Http404() + q = models.ImportLineError.objects.filter(pk=line_id) + if not q.count(): + return + line = q.all()[0] + q = models.Import.query_can_access(request.user).filter(pk=line.import_item_id) + if not q.count(): + raise Http404() + line.ignored = not line.ignored + line.save() + return HttpResponse(content_type="text/plain") + + class PersonCreate(LoginRequiredMixin, CreateView): model = models.Person form_class = forms.BasePersonForm diff --git a/scss/custom.scss b/scss/custom.scss index 3b6d803b3..749d2becf 100644 --- a/scss/custom.scss +++ b/scss/custom.scss @@ -304,6 +304,10 @@ pre { background-color: lighten(red, 40%); } +.import-line-ignored { + color: $gray-500; +} + #import-container { overflow: scroll; -- cgit v1.2.3