From 89b1e4a21e80d53be0e28df8723eec36358034f9 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Thu, 19 Oct 2023 13:32:28 +0200 Subject: ✨ imports: manage ods, xls and xlsx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert them to CSV and store the resulting file in a specific field. --- .../0113_contextrecord_imports_updated.py | 4 +- .../migrations/0112_file_imports_updated.py | 4 +- .../migrations/0113_auto_20231005_1650.py | 39 ----- .../migrations/0113_auto_20231018_1553.py | 39 +++++ .../migrations/0113_auto_20231005_1650.py | 39 ----- .../migrations/0113_auto_20231018_1551.py | 39 +++++ .../migrations/0119_auto_20231005_1650.py | 24 --- .../migrations/0119_auto_20231018_1553.py | 24 +++ ishtar_common/forms_common.py | 31 +++- ishtar_common/libreoffice.py | 9 +- .../migrations/0230_auto_20231005_1650.py | 178 -------------------- .../migrations/0230_auto_20231018_1551.py | 183 +++++++++++++++++++++ .../migrations/0231_default_mandatory_keys.py | 2 +- ishtar_common/models_imports.py | 106 +++++++++--- ishtar_common/templates/ishtar/import_table.html | 4 +- ishtar_common/views.py | 4 +- 16 files changed, 409 insertions(+), 320 deletions(-) delete mode 100644 archaeological_finds/migrations/0113_auto_20231005_1650.py create mode 100644 archaeological_finds/migrations/0113_auto_20231018_1553.py delete mode 100644 archaeological_operations/migrations/0113_auto_20231005_1650.py create mode 100644 archaeological_operations/migrations/0113_auto_20231018_1551.py delete mode 100644 archaeological_warehouse/migrations/0119_auto_20231005_1650.py create mode 100644 archaeological_warehouse/migrations/0119_auto_20231018_1553.py delete mode 100644 ishtar_common/migrations/0230_auto_20231005_1650.py create mode 100644 ishtar_common/migrations/0230_auto_20231018_1551.py diff --git a/archaeological_context_records/migrations/0113_contextrecord_imports_updated.py b/archaeological_context_records/migrations/0113_contextrecord_imports_updated.py index fc365c6a6..6c9a5e085 100644 --- a/archaeological_context_records/migrations/0113_contextrecord_imports_updated.py +++ b/archaeological_context_records/migrations/0113_contextrecord_imports_updated.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2.24 on 2023-10-05 16:50 +# Generated by Django 2.2.24 on 2023-10-18 15:51 from django.db import migrations, models @@ -6,7 +6,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('ishtar_common', '0230_auto_20231005_1650'), + ('ishtar_common', '0230_auto_20231018_1551'), ('archaeological_context_records', '0112_migrate_created'), ] diff --git a/archaeological_files/migrations/0112_file_imports_updated.py b/archaeological_files/migrations/0112_file_imports_updated.py index b44132649..d227e0487 100644 --- a/archaeological_files/migrations/0112_file_imports_updated.py +++ b/archaeological_files/migrations/0112_file_imports_updated.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2.24 on 2023-10-05 16:50 +# Generated by Django 2.2.24 on 2023-10-18 15:53 from django.db import migrations, models @@ -6,7 +6,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('ishtar_common', '0230_auto_20231005_1650'), + ('ishtar_common', '0230_auto_20231018_1551'), ('archaeological_files', '0111_migrate_created'), ] diff --git a/archaeological_finds/migrations/0113_auto_20231005_1650.py b/archaeological_finds/migrations/0113_auto_20231005_1650.py deleted file mode 100644 index 0c761315b..000000000 --- a/archaeological_finds/migrations/0113_auto_20231005_1650.py +++ /dev/null @@ -1,39 +0,0 @@ -# Generated by Django 2.2.24 on 2023-10-05 16:50 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ishtar_common', '0230_auto_20231005_1650'), - ('archaeological_finds', '0112_migrate_created'), - ] - - operations = [ - migrations.AddField( - model_name='basefind', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_finds_basefind', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='find', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_finds_find', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='property', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_finds_property', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='treatment', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_finds_treatment', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='treatmentfile', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_finds_treatmentfile', to='ishtar_common.Import'), - ), - ] diff --git a/archaeological_finds/migrations/0113_auto_20231018_1553.py b/archaeological_finds/migrations/0113_auto_20231018_1553.py new file mode 100644 index 000000000..e9bfad7fd --- /dev/null +++ b/archaeological_finds/migrations/0113_auto_20231018_1553.py @@ -0,0 +1,39 @@ +# Generated by Django 2.2.24 on 2023-10-18 15:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0230_auto_20231018_1551'), + ('archaeological_finds', '0112_migrate_created'), + ] + + operations = [ + migrations.AddField( + model_name='basefind', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_finds_basefind', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='find', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_finds_find', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='property', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_finds_property', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='treatment', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_finds_treatment', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='treatmentfile', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_finds_treatmentfile', to='ishtar_common.Import'), + ), + ] diff --git a/archaeological_operations/migrations/0113_auto_20231005_1650.py b/archaeological_operations/migrations/0113_auto_20231005_1650.py deleted file mode 100644 index 581ec6c9a..000000000 --- a/archaeological_operations/migrations/0113_auto_20231005_1650.py +++ /dev/null @@ -1,39 +0,0 @@ -# Generated by Django 2.2.24 on 2023-10-05 16:50 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ishtar_common', '0230_auto_20231005_1650'), - ('archaeological_operations', '0112_migrate_created'), - ] - - operations = [ - migrations.AddField( - model_name='administrativeact', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_operations_administrativeact', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='archaeologicalsite', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_operations_archaeologicalsite', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='operation', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_operations_operation', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='parcel', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_operations_parcel', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='parcelowner', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_operations_parcelowner', to='ishtar_common.Import'), - ), - ] diff --git a/archaeological_operations/migrations/0113_auto_20231018_1551.py b/archaeological_operations/migrations/0113_auto_20231018_1551.py new file mode 100644 index 000000000..11d1143d0 --- /dev/null +++ b/archaeological_operations/migrations/0113_auto_20231018_1551.py @@ -0,0 +1,39 @@ +# Generated by Django 2.2.24 on 2023-10-18 15:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0230_auto_20231018_1551'), + ('archaeological_operations', '0112_migrate_created'), + ] + + operations = [ + migrations.AddField( + model_name='administrativeact', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_operations_administrativeact', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='archaeologicalsite', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_operations_archaeologicalsite', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='operation', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_operations_operation', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='parcel', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_operations_parcel', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='parcelowner', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_operations_parcelowner', to='ishtar_common.Import'), + ), + ] diff --git a/archaeological_warehouse/migrations/0119_auto_20231005_1650.py b/archaeological_warehouse/migrations/0119_auto_20231005_1650.py deleted file mode 100644 index 7f7cdd749..000000000 --- a/archaeological_warehouse/migrations/0119_auto_20231005_1650.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 2.2.24 on 2023-10-05 16:50 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ishtar_common', '0230_auto_20231005_1650'), - ('archaeological_warehouse', '0118_auto_20230807_1106'), - ] - - operations = [ - migrations.AddField( - model_name='container', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_warehouse_container', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='warehouse', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_warehouse_warehouse', to='ishtar_common.Import'), - ), - ] diff --git a/archaeological_warehouse/migrations/0119_auto_20231018_1553.py b/archaeological_warehouse/migrations/0119_auto_20231018_1553.py new file mode 100644 index 000000000..b84564711 --- /dev/null +++ b/archaeological_warehouse/migrations/0119_auto_20231018_1553.py @@ -0,0 +1,24 @@ +# Generated by Django 2.2.24 on 2023-10-18 15:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0230_auto_20231018_1551'), + ('archaeological_warehouse', '0118_auto_20230807_1106'), + ] + + operations = [ + migrations.AddField( + model_name='container', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_warehouse_container', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='warehouse', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_archaeological_warehouse_warehouse', to='ishtar_common.Import'), + ), + ] diff --git a/ishtar_common/forms_common.py b/ishtar_common/forms_common.py index 5ea90a8d5..dd3a83971 100644 --- a/ishtar_common/forms_common.py +++ b/ishtar_common/forms_common.py @@ -274,16 +274,23 @@ class BaseImportForm(IshtarForm, forms.ModelForm): BAD_CHARS = ["é", "³", "ô", "Ã\xa0", "é"] - def _clean_csv(self, is_csv=False): + def _clean_imported_file(self, types=None): imported_file = self.cleaned_data.get("imported_file", None) + if not imported_file: + return + imported_file_name = imported_file.name.lower() + if types: + if not [1 for tpe in types if imported_file_name.endswith(tpe)]: + if len(types) == 1: + msg = str(_("Bad format. Extension of the file must be {}.")).format(types[0][1:]) + else: + msg = str(_("Bad format. Extension of the file must be: {} or {}.")).format( + ", ".join([tpe[1:] for tpe in types[:-1]]), types[-1][1:] + ) + raise forms.ValidationError(msg) encoding = self.cleaned_data.get("encoding", None) - if imported_file and encoding: + if encoding and imported_file_name.endswith(".csv"): try: - if not imported_file.name.lower().endswith(".csv"): - if is_csv: - raise ValueError() - else: - return imported_file.seek(0) reader = csv.reader(StringIO(imported_file.read().decode(encoding))) idx = 0 @@ -375,7 +382,10 @@ class NewImportForm(BaseImportForm): raise forms.ValidationError( _('"Associated images" field must be a valid zip file.') ) - self._clean_csv(is_csv=True) + types = [".csv"] + if settings.USE_LIBREOFFICE: + types += [".ods", ".xls", ".xlsx"] + self._clean_imported_file(types=types) archive_required = self._need_archive(data) if archive_required and ( not data.get("imported_images", None) @@ -466,7 +476,10 @@ class NewImportGISForm(BaseImportForm): def clean(self): data = super().clean() - self._clean_csv() + types = ["zip", "gpkg", "csv"] + if settings.USE_LIBREOFFICE: + types += [".ods", ".xls", ".xlsx"] + self._clean_imported_file(types=types) return data def save(self, user, commit=True): diff --git a/ishtar_common/libreoffice.py b/ishtar_common/libreoffice.py index 7dbb36e30..3869ca81a 100644 --- a/ishtar_common/libreoffice.py +++ b/ishtar_common/libreoffice.py @@ -1,6 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- - import time import uno @@ -9,14 +8,14 @@ from com.sun.star.beans import PropertyValue from com.sun.star.connection import NoConnectException from com.sun.star.sheet.ValidationType import LIST -from com.sun.star.table import CellRangeAddress, CellAddress +#from com.sun.star.table import CellRangeAddress, CellAddress from ishtar_common.utils import num2col from django.conf import settings -RETRY = 1 +RETRY = 5 class UnoClient: @@ -45,7 +44,7 @@ class UnoClient: if self.remote_context and self.desktop: return try_nb = 0 - while not self.service_manager or try_nb > RETRY: + while not self.service_manager and try_nb <= RETRY: self.connect() try_nb += 1 if not self.service_manager: @@ -64,6 +63,8 @@ class UnoClient: def get_document(self, filename, propval=None): self.create_context() + if not self.remote_context or not self.desktop: + return url = "file://{}".format(filename) if not propval: propval = () diff --git a/ishtar_common/migrations/0230_auto_20231005_1650.py b/ishtar_common/migrations/0230_auto_20231005_1650.py deleted file mode 100644 index 9eb1f478a..000000000 --- a/ishtar_common/migrations/0230_auto_20231005_1650.py +++ /dev/null @@ -1,178 +0,0 @@ -# Generated by Django 2.2.24 on 2023-10-05 16:50 - -import django.core.validators -from django.db import migrations, models -import django.db.models.deletion -import ishtar_common.utils - - -class Migration(migrations.Migration): - - dependencies = [ - ('ishtar_common', '0229_auto_20230608_1303'), - ] - - operations = [ - migrations.CreateModel( - name='ImporterGroup', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=200, verbose_name='Name')), - ('slug', models.SlugField(max_length=100, unique=True, verbose_name='Slug')), - ('description', models.TextField(blank=True, default='', verbose_name='Description')), - ('available', models.BooleanField(default=True, verbose_name='Available')), - ('users', models.ManyToManyField(blank=True, to='ishtar_common.IshtarUser', verbose_name='Users')), - ], - options={ - 'verbose_name': 'Importer - Group', - 'verbose_name_plural': 'Importer - Groups', - 'ordering': ('name',), - }, - ), - migrations.AlterModelOptions( - name='import', - options={'permissions': (('view_own_import', 'Can view own Import'), ('add_own_import', 'Can add own Import'), ('change_own_import', 'Can change own Import'), ('delete_own_import', 'Can delete own Import')), 'verbose_name': 'Import - Import', 'verbose_name_plural': 'Import - Imports'}, - ), - migrations.AlterModelOptions( - name='itemkey', - options={'verbose_name': 'Import - Item key', 'verbose_name_plural': 'Import - Item keys'}, - ), - migrations.AddField( - model_name='document', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_document', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='geovectordata', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_geovectordata', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='import', - name='next_import', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='imports', to='ishtar_common.Import', verbose_name='Next import'), - ), - migrations.AddField( - model_name='importerdefault', - name='required_fields', - field=models.CharField(blank=True, default='', help_text='Theses defaults values only apply if the designated fields are not empty. Use the "__" notation to pass between models.Leave empty to always apply.', max_length=500, verbose_name='Required fields'), - ), - migrations.AddField( - model_name='importertype', - name='archive_required', - field=models.BooleanField(default=False, verbose_name='Archive required'), - ), - migrations.AddField( - model_name='importertype', - name='ignore_errors', - field=models.TextField(blank=True, help_text='If an error is encountered with the following character strings, the error is not reported in the error file. Each message is separated with a line break.', null=True, verbose_name='Error messages to ignore'), - ), - migrations.AddField( - model_name='importertype', - name='is_import', - field=models.BooleanField(default=True, verbose_name='Can be import'), - ), - migrations.AddField( - model_name='importertype', - name='pre_import_message', - field=models.TextField(blank=True, default='', max_length=500, verbose_name='Pre-import form message'), - ), - migrations.AddField( - model_name='importertype', - name='tab_number', - field=models.PositiveIntegerField(default=1, help_text='When using an Excel or Calc file choose the tab number. Keep it to 1 by default.', validators=[django.core.validators.MinValueValidator(1)], verbose_name='Tab number'), - ), - migrations.AddField( - model_name='organization', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_organization', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='person', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_person', to='ishtar_common.Import'), - ), - migrations.AddField( - model_name='town', - name='imports_updated', - field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_town', to='ishtar_common.Import'), - ), - migrations.AlterField( - model_name='import', - name='imported_images', - field=models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Associated documents (zip file)'), - ), - migrations.AlterField( - model_name='importercolumn', - name='col_number', - field=models.SmallIntegerField(default=1, help_text='Column number in the table. Put 0 or negative number for pre-importer field.', verbose_name='Column number'), - ), - migrations.AlterField( - model_name='importerdefault', - name='target', - field=models.CharField(help_text='The target of the default values. Can be set to empty with "-". Use the "__" notation to pass between models.', max_length=500, verbose_name='Target'), - ), - migrations.AlterField( - model_name='ishtarsiteprofile', - name='account_naming_style', - field=models.CharField(choices=[('NF', 'name.firstname'), ('FN', 'firstname.name')], default='FN', max_length=2, verbose_name='Naming style for accounts'), - ), - migrations.CreateModel( - name='ImportGroup', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=500, null=True, verbose_name='Name')), - ('imported_file', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Imported file')), - ('imported_images', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Associated documents (zip file)')), - ('archive_file', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=255, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Archive file')), - ('encoding', models.CharField(choices=[('windows-1252', 'windows-1252'), ('ISO-8859-15', 'ISO-8859-15'), ('utf-8', 'utf-8')], default='utf-8', help_text='Only required for CSV file', max_length=15, verbose_name='Encoding')), - ('csv_sep', models.CharField(choices=[(',', ','), (';', ';'), ('|', '|')], default=',', help_text='Separator for CSV file. Standard is comma but Microsoft Excel do not follow this standard and use semi-colon.', max_length=1, verbose_name='CSV separator')), - ('skip_lines', models.IntegerField(default=1, help_text='Number of header lines in your file (can be 0 and should be 0 for geopackage or Shapefile).', verbose_name='Skip lines')), - ('creation_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Creation date')), - ('end_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='End date')), - ('current_import', models.PositiveIntegerField(blank=True, null=True, verbose_name='Current import')), - ('state', models.CharField(choices=[('C', 'Created'), ('AP', 'Analyse in progress'), ('A', 'Analysed'), ('IQ', 'Import in queue'), ('IP', 'Import in progress'), ('PP', 'Post-processing in progress'), ('FE', 'Finished with errors'), ('F', 'Finished'), ('AC', 'Archived')], default='C', max_length=2, verbose_name='State')), - ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ImporterGroup', verbose_name='Importer group type')), - ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.IshtarUser')), - ], - options={ - 'verbose_name': 'Import - Group', - 'verbose_name_plural': 'Import - Groups', - 'permissions': (('view_own_importgroup', 'Can view own Import Group'), ('add_own_importgroup', 'Can add own Import Group'), ('change_own_importgroup', 'Can change own Import Group'), ('delete_own_importgroup', 'Can delete own Import Group')), - }, - bases=(models.Model, ishtar_common.utils.OwnPerms, ishtar_common.utils.SheetItem), - ), - migrations.AddField( - model_name='import', - name='group', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='imports', to='ishtar_common.ImportGroup', verbose_name='Group'), - ), - migrations.CreateModel( - name='ImporterGroupImporter', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('order', models.PositiveIntegerField(default=10, validators=[django.core.validators.MinValueValidator(1)], verbose_name='Order')), - ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='importer_types', to='ishtar_common.ImporterGroup')), - ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='ishtar_common.ImporterType')), - ], - options={ - 'verbose_name': 'Importer - Group <-> Importer', - 'ordering': ('group', 'order'), - 'unique_together': {('group', 'order')}, - }, - ), - migrations.CreateModel( - name='ImportColumnValue', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('value', models.TextField(blank=True, default='')), - ('column', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ImporterColumn')), - ('import_item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.Import')), - ], - options={ - 'verbose_name': 'Import - Pre-import value', - 'verbose_name_plural': 'Import - Pre-import values', - 'unique_together': {('column', 'import_item')}, - }, - ), - ] diff --git a/ishtar_common/migrations/0230_auto_20231018_1551.py b/ishtar_common/migrations/0230_auto_20231018_1551.py new file mode 100644 index 000000000..c2fc2abe0 --- /dev/null +++ b/ishtar_common/migrations/0230_auto_20231018_1551.py @@ -0,0 +1,183 @@ +# Generated by Django 2.2.24 on 2023-10-18 15:51 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import ishtar_common.utils + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0229_auto_20230608_1303'), + ] + + operations = [ + migrations.CreateModel( + name='ImporterGroup', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=200, verbose_name='Name')), + ('slug', models.SlugField(max_length=100, unique=True, verbose_name='Slug')), + ('description', models.TextField(blank=True, default='', verbose_name='Description')), + ('available', models.BooleanField(default=True, verbose_name='Available')), + ('users', models.ManyToManyField(blank=True, to='ishtar_common.IshtarUser', verbose_name='Users')), + ], + options={ + 'verbose_name': 'Importer - Group', + 'verbose_name_plural': 'Importer - Groups', + 'ordering': ('name',), + }, + ), + migrations.AlterModelOptions( + name='import', + options={'permissions': (('view_own_import', 'Can view own Import'), ('add_own_import', 'Can add own Import'), ('change_own_import', 'Can change own Import'), ('delete_own_import', 'Can delete own Import')), 'verbose_name': 'Import - Import', 'verbose_name_plural': 'Import - Imports'}, + ), + migrations.AlterModelOptions( + name='itemkey', + options={'verbose_name': 'Import - Item key', 'verbose_name_plural': 'Import - Item keys'}, + ), + migrations.AddField( + model_name='document', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_document', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='geovectordata', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_geovectordata', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='import', + name='imported_values', + field=models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Imported values'), + ), + migrations.AddField( + model_name='import', + name='next_import', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='imports', to='ishtar_common.Import', verbose_name='Next import'), + ), + migrations.AddField( + model_name='importerdefault', + name='required_fields', + field=models.CharField(blank=True, default='', help_text='Theses defaults values only apply if the designated fields are not empty. Use the "__" notation to pass between models.Leave empty to always apply.', max_length=500, verbose_name='Required fields'), + ), + migrations.AddField( + model_name='importertype', + name='archive_required', + field=models.BooleanField(default=False, verbose_name='Archive required'), + ), + migrations.AddField( + model_name='importertype', + name='ignore_errors', + field=models.TextField(blank=True, help_text='If an error is encountered with the following character strings, the error is not reported in the error file. Each message is separated with a line break.', null=True, verbose_name='Error messages to ignore'), + ), + migrations.AddField( + model_name='importertype', + name='is_import', + field=models.BooleanField(default=True, verbose_name='Can be import'), + ), + migrations.AddField( + model_name='importertype', + name='pre_import_message', + field=models.TextField(blank=True, default='', max_length=500, verbose_name='Pre-import form message'), + ), + migrations.AddField( + model_name='importertype', + name='tab_number', + field=models.PositiveIntegerField(default=1, help_text='When using an Excel or Calc file choose the tab number. Keep it to 1 by default.', validators=[django.core.validators.MinValueValidator(1)], verbose_name='Tab number'), + ), + migrations.AddField( + model_name='organization', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_organization', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='person', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_person', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='town', + name='imports_updated', + field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_town', to='ishtar_common.Import'), + ), + migrations.AlterField( + model_name='import', + name='imported_images', + field=models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Associated documents (zip file)'), + ), + migrations.AlterField( + model_name='importercolumn', + name='col_number', + field=models.SmallIntegerField(default=1, help_text='Column number in the table. Put 0 or negative number for pre-importer field.', verbose_name='Column number'), + ), + migrations.AlterField( + model_name='importerdefault', + name='target', + field=models.CharField(help_text='The target of the default values. Can be set to empty with "-". Use the "__" notation to pass between models.', max_length=500, verbose_name='Target'), + ), + migrations.AlterField( + model_name='ishtarsiteprofile', + name='account_naming_style', + field=models.CharField(choices=[('NF', 'name.firstname'), ('FN', 'firstname.name')], default='FN', max_length=2, verbose_name='Naming style for accounts'), + ), + migrations.CreateModel( + name='ImportGroup', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=500, null=True, verbose_name='Name')), + ('imported_file', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Imported file')), + ('imported_images', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Associated documents (zip file)')), + ('archive_file', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=255, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Archive file')), + ('encoding', models.CharField(choices=[('windows-1252', 'windows-1252'), ('ISO-8859-15', 'ISO-8859-15'), ('utf-8', 'utf-8')], default='utf-8', help_text='Only required for CSV file', max_length=15, verbose_name='Encoding')), + ('csv_sep', models.CharField(choices=[(',', ','), (';', ';'), ('|', '|')], default=',', help_text='Separator for CSV file. Standard is comma but Microsoft Excel do not follow this standard and use semi-colon.', max_length=1, verbose_name='CSV separator')), + ('skip_lines', models.IntegerField(default=1, help_text='Number of header lines in your file (can be 0 and should be 0 for geopackage or Shapefile).', verbose_name='Skip lines')), + ('creation_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Creation date')), + ('end_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='End date')), + ('current_import', models.PositiveIntegerField(blank=True, null=True, verbose_name='Current import')), + ('state', models.CharField(choices=[('C', 'Created'), ('AP', 'Analyse in progress'), ('A', 'Analysed'), ('IQ', 'Import in queue'), ('IP', 'Import in progress'), ('PP', 'Post-processing in progress'), ('FE', 'Finished with errors'), ('F', 'Finished'), ('AC', 'Archived')], default='C', max_length=2, verbose_name='State')), + ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ImporterGroup', verbose_name='Importer group type')), + ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.IshtarUser')), + ], + options={ + 'verbose_name': 'Import - Group', + 'verbose_name_plural': 'Import - Groups', + 'permissions': (('view_own_importgroup', 'Can view own Import Group'), ('add_own_importgroup', 'Can add own Import Group'), ('change_own_importgroup', 'Can change own Import Group'), ('delete_own_importgroup', 'Can delete own Import Group')), + }, + bases=(models.Model, ishtar_common.utils.OwnPerms, ishtar_common.utils.SheetItem), + ), + migrations.AddField( + model_name='import', + name='group', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='imports', to='ishtar_common.ImportGroup', verbose_name='Group'), + ), + migrations.CreateModel( + name='ImporterGroupImporter', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('order', models.PositiveIntegerField(default=10, validators=[django.core.validators.MinValueValidator(1)], verbose_name='Order')), + ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='importer_types', to='ishtar_common.ImporterGroup')), + ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='ishtar_common.ImporterType')), + ], + options={ + 'verbose_name': 'Importer - Group <-> Importer', + 'ordering': ('group', 'order'), + 'unique_together': {('group', 'order')}, + }, + ), + migrations.CreateModel( + name='ImportColumnValue', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('value', models.TextField(blank=True, default='')), + ('column', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ImporterColumn')), + ('import_item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.Import')), + ], + options={ + 'verbose_name': 'Import - Pre-import value', + 'verbose_name_plural': 'Import - Pre-import values', + 'unique_together': {('column', 'import_item')}, + }, + ), + ] diff --git a/ishtar_common/migrations/0231_default_mandatory_keys.py b/ishtar_common/migrations/0231_default_mandatory_keys.py index 59624d00b..568e60b9a 100644 --- a/ishtar_common/migrations/0231_default_mandatory_keys.py +++ b/ishtar_common/migrations/0231_default_mandatory_keys.py @@ -30,7 +30,7 @@ def migrate(apps, __): class Migration(migrations.Migration): dependencies = [ - ('ishtar_common', '0230_auto_20231005_1650'), + ('ishtar_common', '0230_auto_20231018_1551'), ] operations = [ diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py index 3f687b93d..a240f4326 100644 --- a/ishtar_common/models_imports.py +++ b/ishtar_common/models_imports.py @@ -19,6 +19,7 @@ import csv import datetime +import random from pathlib import Path import fiona @@ -1754,6 +1755,7 @@ class ImportGroup(BaseImport): pass setattr(self, attr, None) sub_import_file_attr.append("imported_file") + sub_import_file_attr.append("imported_values") if profile.delete_image_zip_on_archive: sub_import_file_attr.append("imported_images") for sub_import in import_list: @@ -1784,6 +1786,9 @@ class ImportGroup(BaseImport): def get_all_updated(self): return self._get_all_related("import_updated_") + def get_imported_values(self): + return self.imported_file + def save(self, *args, **kwargs): add = self._state.adding super().save(*args, **kwargs) @@ -1838,6 +1843,14 @@ class Import(BaseImport): "If a group is selected, target key saved in this group will be used." ), ) + imported_values = models.FileField( + _("Imported values"), + upload_to="upload/imports/%Y/%m/", + max_length=220, + help_text=max_size_help(), + blank=True, + null=True, + ) error_file = models.FileField( _("Error file"), upload_to="upload/imports/%Y/%m/", @@ -1910,6 +1923,11 @@ class Import(BaseImport): def __str__(self): return "{} | {}".format(self.name or "-", self.importer_type) + def __init__(self, *args, **kwargs): + returned = super().__init__(*args, **kwargs) + self._initial_imported_file = self.imported_file.path if self.imported_file else "" + return returned + def is_available(self, ishtar_user) -> bool: if ishtar_user.is_superuser or self.user == ishtar_user: return True @@ -2060,14 +2078,56 @@ class Import(BaseImport): self._pre_import_values = values return values + def get_imported_values(self): + if self.imported_values: + return self.imported_values + return self.imported_file + + def set_imported_values(self): + if not settings.USE_LIBREOFFICE or not self.imported_file or not UnoCalc: + return + name = self.imported_file.name.lower() + if not name.endswith(".ods") and not name.endswith(".xls") and not name.endswith(".xlsx"): + return + imported_file_path = os.path.abspath(self.imported_file.path) + media_root = os.path.abspath(settings.MEDIA_ROOT) + if not imported_file_path.startswith(media_root): + return + uno = UnoCalc() + + calc = uno.open_calc(imported_file_path) + if not calc: + return + try: + sheet = uno.get_sheet(calc, (self.importer_type.tab_number or 1) - 1) + except Exception: + return + col_numbers = [c.col_number for c in self.importer_type.columns.all()] + if not col_numbers: + return + last_column = max(col_numbers) + filename = ".".join(imported_file_path.split('.')[:-1]) + f"-{random.randint(1, 10000):05d}.csv" + + with open(filename, "w") as result_file: + w = csv.writer(result_file) + w.writerows(data for data in uno.sheet_get_data(sheet, last_column=last_column)) + + name = filename[len(media_root):] + if name.startswith(os.sep): + name = name[1:] + self.imported_values.name = name + self._initial_imported_file = self.imported_file.path + return True + def get_number_of_lines(self): if self.number_of_line: return self.number_of_line if self.importer_type.type == "gis": return - if not self.imported_file or not self.imported_file.path: + imported_values = self.get_imported_values() + if not imported_values or not imported_values.path: return - filename = self.imported_file.path + filename = imported_values.path encodings = [self.encoding] encodings += [coding for coding, c in ENCODINGS if coding != self.encoding] for encoding in encodings: @@ -2194,10 +2254,10 @@ class Import(BaseImport): ) def _data_table_tab(self): - imported_file = self.imported_file.path + imported_values = self.get_imported_values().path tmpdir = None - if zipfile.is_zipfile(imported_file): - z = zipfile.ZipFile(imported_file) + if zipfile.is_zipfile(imported_values): + z = zipfile.ZipFile(imported_values) filename = None for name in z.namelist(): # get first CSV file found @@ -2207,13 +2267,13 @@ class Import(BaseImport): if not filename: return [] tmpdir = tempfile.mkdtemp(prefix="tmp-ishtar-") - imported_file = z.extract(filename, tmpdir) + imported_values = z.extract(filename, tmpdir) encodings = [self.encoding] encodings += [coding for coding, c in ENCODINGS if coding != self.encoding] for encoding in encodings: try: - with open(imported_file, encoding=encoding) as csv_file: + with open(imported_values, encoding=encoding) as csv_file: vals = [ line for line in csv.reader(csv_file, delimiter=self.csv_sep) ] @@ -2233,12 +2293,12 @@ class Import(BaseImport): def _data_table_gis(self, get_gis_attr=False): self.gis_attr = None - imported_file = self.imported_file.path + imported_values = self.get_imported_values().path tmp_dir = None file_type = "gpkg" - if zipfile.is_zipfile(imported_file): - z = zipfile.ZipFile(imported_file) - imported_file = None + if zipfile.is_zipfile(imported_values): + z = zipfile.ZipFile(imported_values) + imported_values = None filenames = [] for name in z.namelist(): # get first CSV file found @@ -2261,22 +2321,22 @@ class Import(BaseImport): if filename.lower().endswith(".shp") or filename.lower().endswith( ".gpkg" ): - imported_file = z.extract(filename, tmp_dir) + imported_values = z.extract(filename, tmp_dir) else: z.extract(filename, tmp_dir) - elif imported_file.endswith(".csv"): + elif imported_values.endswith(".csv"): return self._data_table_tab() - elif not imported_file.endswith(".gpkg"): + elif not imported_values.endswith(".gpkg"): raise ImporterError(_("Invalid GIS file.")) - if not imported_file: + if not imported_values: raise ImporterError(_("Invalid GIS file.")) kwargs = {} if self.importer_type.layer_name: kwargs["layer"] = self.importer_type.layer_name try: - with fiona.open(imported_file, **kwargs) as collection: + with fiona.open(imported_values, **kwargs) as collection: schema = collection.schema geometry = schema["geometry"] if geometry not in IMPORT_GEOMETRY: @@ -2292,7 +2352,7 @@ class Import(BaseImport): if not crs: driver_type = {"shp": "ESRI Shapefile", "gpkg": "GPKG"} driver = ogr.GetDriverByName(driver_type[file_type]) - shape = driver.Open(imported_file) + shape = driver.Open(imported_values) layer = shape.GetLayer() crs = layer.GetSpatialRef() auth = crs.GetAttrValue("AUTHORITY", 0) @@ -2470,7 +2530,7 @@ class Import(BaseImport): ) except IOError: error_message = str(_("Error on imported file: {}")).format( - self.imported_file + self.get_imported_values() ) importer.errors = [error_message] if session_key: @@ -2570,7 +2630,7 @@ class Import(BaseImport): return True def _archive(self): - file_attr = ["imported_file", "error_file", "result_file", "match_file"] + file_attr = ["imported_file", "imported_values", "error_file", "result_file", "match_file"] files = [ (k, getattr(self, k).path, getattr(self, k).name.split(os.sep)[-1]) for k in file_attr @@ -2626,6 +2686,14 @@ class Import(BaseImport): def get_all_updated(self): return self._get_all_related("import_updated_") + def save(self, *args, **kwargs): + if self.imported_file: + if self._initial_imported_file != self.imported_file.path or not self.imported_values: + self.set_imported_values() + elif self.imported_values: + self.imported_values = None + super().save(*args, **kwargs) + def pre_delete_import(sender, **kwargs): # deleted imported items when an import is delete diff --git a/ishtar_common/templates/ishtar/import_table.html b/ishtar_common/templates/ishtar/import_table.html index be3af600b..08c949654 100644 --- a/ishtar_common/templates/ishtar/import_table.html +++ b/ishtar_common/templates/ishtar/import_table.html @@ -88,7 +88,7 @@