From 78f4c43b6e2ce3d9cd22256ee5f8039cf5915ecd Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Thu, 10 Aug 2023 13:23:16 +0200 Subject: ✨ Imports groups: initialize action MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../migrations/0230_auto_20230809_1149.py | 95 ------------------- .../migrations/0230_auto_20230810_1140.py | 105 +++++++++++++++++++++ ishtar_common/models_imports.py | 71 ++++++++++++-- ishtar_common/templates/ishtar/import_table.html | 14 ++- 4 files changed, 180 insertions(+), 105 deletions(-) delete mode 100644 ishtar_common/migrations/0230_auto_20230809_1149.py create mode 100644 ishtar_common/migrations/0230_auto_20230810_1140.py diff --git a/ishtar_common/migrations/0230_auto_20230809_1149.py b/ishtar_common/migrations/0230_auto_20230809_1149.py deleted file mode 100644 index aaf97671a..000000000 --- a/ishtar_common/migrations/0230_auto_20230809_1149.py +++ /dev/null @@ -1,95 +0,0 @@ -# Generated by Django 2.2.24 on 2023-08-09 11:49 - -import django.core.validators -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('ishtar_common', '0229_auto_20230608_1303'), - ] - - operations = [ - migrations.CreateModel( - name='ImporterGroup', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=200, verbose_name='Name')), - ('slug', models.SlugField(max_length=100, unique=True, verbose_name='Slug')), - ('description', models.TextField(blank=True, default='', verbose_name='Description')), - ('available', models.BooleanField(default=True, verbose_name='Available')), - ], - options={ - 'verbose_name': 'Importer - Group', - 'verbose_name_plural': 'Importer - Groups', - 'ordering': ('name',), - }, - ), - migrations.AlterModelOptions( - name='import', - options={'verbose_name': 'Import - Import', 'verbose_name_plural': 'Import - Imports'}, - ), - migrations.AddField( - model_name='importertype', - name='archive_required', - field=models.BooleanField(default=False, verbose_name='Archive required'), - ), - migrations.AddField( - model_name='importertype', - name='is_import', - field=models.BooleanField(default=True, verbose_name='Can be import'), - ), - migrations.AddField( - model_name='importertype', - name='tab_number', - field=models.PositiveIntegerField(default=1, help_text='When using an Excel or Calc file choose the tab number. Keep it to 1 by default.', validators=[django.core.validators.MinValueValidator(1)], verbose_name='Tab number'), - ), - migrations.AlterField( - model_name='ishtarsiteprofile', - name='account_naming_style', - field=models.CharField(choices=[('NF', 'name.firstname'), ('FN', 'firstname.name')], default='FN', max_length=2, verbose_name='Naming style for accounts'), - ), - migrations.CreateModel( - name='ImportGroup', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=500, null=True, verbose_name='Name')), - ('imported_file', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Imported file')), - ('imported_images', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Associated images (zip file)')), - ('encoding', models.CharField(choices=[('windows-1252', 'windows-1252'), ('ISO-8859-15', 'ISO-8859-15'), ('utf-8', 'utf-8')], default='utf-8', help_text='Only required for CSV file', max_length=15, verbose_name='Encoding')), - ('csv_sep', models.CharField(choices=[(',', ','), (';', ';'), ('|', '|')], default=',', help_text='Separator for CSV file. Standard is comma but Microsoft Excel do not follow this standard and use semi-colon.', max_length=1, verbose_name='CSV separator')), - ('skip_lines', models.IntegerField(default=1, help_text='Number of header lines in your file (can be 0 and should be 0 for geopackage or Shapefile).', verbose_name='Skip lines')), - ('creation_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Creation date')), - ('end_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='End date')), - ('current_import', models.PositiveIntegerField(blank=True, null=True, verbose_name='Current import')), - ('state', models.CharField(choices=[('C', 'Created'), ('AP', 'Analyse in progress'), ('A', 'Analysed'), ('IQ', 'Import in queue'), ('IP', 'Import in progress'), ('PP', 'Post-processing in progress'), ('FE', 'Finished with errors'), ('F', 'Finished'), ('AC', 'Archived')], default='C', max_length=2, verbose_name='State')), - ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ImporterGroup', verbose_name='Importer group type')), - ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.IshtarUser')), - ], - options={ - 'verbose_name': 'Import - Group', - 'verbose_name_plural': 'Import - Groups', - }, - ), - migrations.AddField( - model_name='import', - name='group', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='imports', to='ishtar_common.ImportGroup', verbose_name='Group'), - ), - migrations.CreateModel( - name='ImporterGroupImporter', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('order', models.PositiveIntegerField(default=10, validators=[django.core.validators.MinValueValidator(1)], verbose_name='Order')), - ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='importer_types', to='ishtar_common.ImporterGroup')), - ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='ishtar_common.ImporterType')), - ], - options={ - 'verbose_name': 'Importer - Group <-> Importer', - 'ordering': ('group', 'order'), - 'unique_together': {('group', 'order')}, - }, - ), - ] diff --git a/ishtar_common/migrations/0230_auto_20230810_1140.py b/ishtar_common/migrations/0230_auto_20230810_1140.py new file mode 100644 index 000000000..b668baa84 --- /dev/null +++ b/ishtar_common/migrations/0230_auto_20230810_1140.py @@ -0,0 +1,105 @@ +# Generated by Django 2.2.24 on 2023-08-10 11:40 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0229_auto_20230608_1303'), + ] + + operations = [ + migrations.CreateModel( + name='ImporterGroup', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=200, verbose_name='Name')), + ('slug', models.SlugField(max_length=100, unique=True, verbose_name='Slug')), + ('description', models.TextField(blank=True, default='', verbose_name='Description')), + ('available', models.BooleanField(default=True, verbose_name='Available')), + ], + options={ + 'verbose_name': 'Importer - Group', + 'verbose_name_plural': 'Importer - Groups', + 'ordering': ('name',), + }, + ), + migrations.AlterModelOptions( + name='import', + options={'verbose_name': 'Import - Import', 'verbose_name_plural': 'Import - Imports'}, + ), + migrations.AddField( + model_name='import', + name='next_import', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='imports', to='ishtar_common.Import', verbose_name='Next import'), + ), + migrations.AddField( + model_name='importertype', + name='archive_required', + field=models.BooleanField(default=False, verbose_name='Archive required'), + ), + migrations.AddField( + model_name='importertype', + name='is_import', + field=models.BooleanField(default=True, verbose_name='Can be import'), + ), + migrations.AddField( + model_name='importertype', + name='tab_number', + field=models.PositiveIntegerField(default=1, help_text='When using an Excel or Calc file choose the tab number. Keep it to 1 by default.', validators=[django.core.validators.MinValueValidator(1)], verbose_name='Tab number'), + ), + migrations.AlterField( + model_name='import', + name='imported_images', + field=models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Associated documents (zip file)'), + ), + migrations.AlterField( + model_name='ishtarsiteprofile', + name='account_naming_style', + field=models.CharField(choices=[('NF', 'name.firstname'), ('FN', 'firstname.name')], default='FN', max_length=2, verbose_name='Naming style for accounts'), + ), + migrations.CreateModel( + name='ImportGroup', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=500, null=True, verbose_name='Name')), + ('imported_file', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Imported file')), + ('imported_images', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Associated documents (zip file)')), + ('encoding', models.CharField(choices=[('windows-1252', 'windows-1252'), ('ISO-8859-15', 'ISO-8859-15'), ('utf-8', 'utf-8')], default='utf-8', help_text='Only required for CSV file', max_length=15, verbose_name='Encoding')), + ('csv_sep', models.CharField(choices=[(',', ','), (';', ';'), ('|', '|')], default=',', help_text='Separator for CSV file. Standard is comma but Microsoft Excel do not follow this standard and use semi-colon.', max_length=1, verbose_name='CSV separator')), + ('skip_lines', models.IntegerField(default=1, help_text='Number of header lines in your file (can be 0 and should be 0 for geopackage or Shapefile).', verbose_name='Skip lines')), + ('creation_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Creation date')), + ('end_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='End date')), + ('current_import', models.PositiveIntegerField(blank=True, null=True, verbose_name='Current import')), + ('state', models.CharField(choices=[('C', 'Created'), ('AP', 'Analyse in progress'), ('A', 'Analysed'), ('IQ', 'Import in queue'), ('IP', 'Import in progress'), ('PP', 'Post-processing in progress'), ('FE', 'Finished with errors'), ('F', 'Finished'), ('AC', 'Archived')], default='C', max_length=2, verbose_name='State')), + ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ImporterGroup', verbose_name='Importer group type')), + ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.IshtarUser')), + ], + options={ + 'verbose_name': 'Import - Group', + 'verbose_name_plural': 'Import - Groups', + }, + ), + migrations.AddField( + model_name='import', + name='group', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='imports', to='ishtar_common.ImportGroup', verbose_name='Group'), + ), + migrations.CreateModel( + name='ImporterGroupImporter', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('order', models.PositiveIntegerField(default=10, validators=[django.core.validators.MinValueValidator(1)], verbose_name='Order')), + ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='importer_types', to='ishtar_common.ImporterGroup')), + ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='ishtar_common.ImporterType')), + ], + options={ + 'verbose_name': 'Importer - Group <-> Importer', + 'ordering': ('group', 'order'), + 'unique_together': {('group', 'order')}, + }, + ), + ] diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py index d24317c2f..718f76ef3 100644 --- a/ishtar_common/models_imports.py +++ b/ishtar_common/models_imports.py @@ -19,6 +19,8 @@ import csv import datetime +import sys + import fiona from fiona import crs as fiona_crs import json @@ -1347,7 +1349,7 @@ class ImportGroup(BaseImport): actions.append(("I", _("Re-import"))) actions.append(("AC", _("Archive"))) if self.state == "AC": - state = "FE" if any([1 for imp in self.imports.all() if imp.error_file]) else "F" + state = "FE" if any(1 for imp in self.imports.all() if imp.error_file) else "F" actions.append((state, _("Unarchive"))) actions.append(("D", _("Delete"))) return actions @@ -1362,6 +1364,37 @@ class ImportGroup(BaseImport): self.end_date = datetime.datetime.now() self.save() + def importation( + self, + session_key=None, + line_to_process=None, + simulate=False, + return_importer_and_data=False, + request=None, + ): + q = self.imports + if not q.count(): + return + self.state = "IP" + self.end_date = datetime.datetime.now() + self.save() + first = self.import_list()[0] + first.importation( + session_key=session_key, + line_to_process=line_to_process, + simulate=simulate, + return_importer_and_data=return_importer_and_data, + request=request + ) + # from the first import if all is good, cascade import + has_error = any(i.error_file.name for i in q.all() if i.error_file.name) + if has_error: + self.state = "FE" + else: + self.state = "F" + self.end_date = datetime.datetime.now() + self.save() + def get_all_imported(self): imported = [] for imp in self.imports.all(): @@ -1376,27 +1409,39 @@ class ImportGroup(BaseImport): if not add: return name = f"{self.name} ({self.importer_type.name})" + imports = [] + imported_file, imported_images = None, None + if self.imported_file: + imported_file = ContentFile(self.imported_file.read()) + imported_file.name = self.imported_file.name.split(os.sep)[-1] + if self.imported_images: + imported_images = ContentFile(self.imported_images.read()) + imported_images.name = self.imported_images.name.split(os.sep)[-1] + for import_type_relation in self.importer_type.importer_types.all(): import_type = import_type_relation.importer_type imp = Import.objects.create( name=name, importer_type=import_type, - group=self + group=self, ) + imports.append(imp) modified = False # TODO: only get the relevant sheet - if self.imported_file: - imported_file = ContentFile(self.imported_file.read()) - imported_file.name = self.imported_file.name + if imported_file: imp.imported_file = imported_file modified = True - if import_type.archive_required and self.imported_images: - imported_image = ContentFile(self.imported_images.read()) - imported_image.name = self.imported_images.name - imp.imported_images = imported_image + if import_type.archive_required and imported_images: + imp.imported_images = imported_images modified = True if modified: imp.save() + previous = None + for imp in reversed(imports): + if previous: + imp.next_import = previous + imp.save() + previous = imp class Import(BaseImport): @@ -1472,6 +1517,9 @@ class Import(BaseImport): ) group = models.ForeignKey(ImportGroup, blank=True, null=True, on_delete=models.CASCADE, verbose_name=_("Group"), related_name="imports") + next_import = models.ForeignKey( + "Import", blank=True, null=True, on_delete=models.SET_NULL, + verbose_name=_("Next import"), related_name="imports") class Meta: verbose_name = _("Import - Import") @@ -1967,6 +2015,11 @@ class Import(BaseImport): ) self.end_date = datetime.datetime.now() self.save() + if self.next_import and not importer.errors: + return self.next_import.importation( + session_key=session_key, line_to_process=line_to_process, simulate=simulate, + return_importer_and_data=return_importer_and_data, request=request + ) if return_importer_and_data: return importer, data diff --git a/ishtar_common/templates/ishtar/import_table.html b/ishtar_common/templates/ishtar/import_table.html index 39b8e0494..138dbba65 100644 --- a/ishtar_common/templates/ishtar/import_table.html +++ b/ishtar_common/templates/ishtar/import_table.html @@ -87,7 +87,7 @@ $("#import-list").find('input').prop('disabled', true); {% trans "File" context "not a directory" %} {% endif %} - {% if import.state == 'IP' or import.state == 'PP' and import.current_line %} + {% if import.importer_type.type_label and import.state == 'IP' or import.state == 'PP' and import.current_line %}
@@ -128,6 +128,18 @@ $("#import-list").find('input').prop('disabled', true); {% trans "File" context "not a directory" %} {% endif %} + {% if sub.state == 'IP' or sub.state == 'PP' %} + + +
+
+  {{sub.current_line}} / {{sub.number_of_line}} +
+
+ + + {% endif %} {% endfor %} {% endif %} {% endfor %} -- cgit v1.2.3