From e8aa8fdb0ea4f44266368df0817351fd6e965df8 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Tue, 8 Aug 2023 17:56:48 +0200 Subject: ✨ Imports groups: archive_required field - automatically create related imports on group creation - adapt import list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ishtar_common/forms_common.py | 38 +++++++-- .../migrations/0230_auto_20230807_1105.py | 88 -------------------- .../migrations/0230_auto_20230809_1149.py | 95 ++++++++++++++++++++++ ishtar_common/models.py | 1 - ishtar_common/models_imports.py | 37 ++++++++- ishtar_common/templates/ishtar/import_table.html | 43 ++++++++++ ishtar_common/views.py | 2 +- 7 files changed, 202 insertions(+), 102 deletions(-) delete mode 100644 ishtar_common/migrations/0230_auto_20230807_1105.py create mode 100644 ishtar_common/migrations/0230_auto_20230809_1149.py diff --git a/ishtar_common/forms_common.py b/ishtar_common/forms_common.py index 6a351ab3c..d2011338b 100644 --- a/ishtar_common/forms_common.py +++ b/ishtar_common/forms_common.py @@ -250,6 +250,7 @@ class BaseImportForm(IshtarForm, forms.ModelForm): self.fields["importer_type"].choices = [("", "--")] + [ (imp.pk, imp.name) for imp in models.ImporterType.objects.filter(available=True, + is_import=True, type=self.importer_type) ] @@ -309,9 +310,20 @@ class BaseImportForm(IshtarForm, forms.ModelForm): return data +def get_archive_from_link(archive_link): + request = requests.get(archive_link, stream=True) + ntf = tempfile.NamedTemporaryFile() + for block in request.iter_content(1024 * 8): + if not block: + break + ntf.write(block) + file_name = archive_link.split("/")[-1] + return file_name, ntf + + class NewImportForm(BaseImportForm): imported_images_link = forms.URLField( - label=_("Associated images (web link to a zip file)"), required=False + label=_("Associated documents (web link to a zip file)"), required=False ) class Meta: @@ -334,6 +346,10 @@ class NewImportForm(BaseImportForm): "imported_images": FormHeader(_("Documents/Images")), } + def _need_archive(self, data): + tpe = data["importer_type"] + return tpe.archive_required + def clean(self): data = super().clean() if data.get("imported_images_link", None) and data.get("imported_images", None): @@ -353,6 +369,12 @@ class NewImportForm(BaseImportForm): _("\"Associated images\" field must be a valid zip file.") ) self._clean_csv(is_csv=True) + archive_required = self._need_archive(data) + if archive_required and ( + not data.get("imported_images", None) and + not data.get("imported_images_link", None) + ): + raise forms.ValidationError(_("This importer need a document archive.")) return data def clean_imported_images_link(self): @@ -377,14 +399,8 @@ class NewImportForm(BaseImportForm): item = super().save(commit) if not imported_images_link: return item - request = requests.get(imported_images_link, stream=True) - ntf = tempfile.NamedTemporaryFile() - for block in request.iter_content(1024 * 8): - if not block: - break - ntf.write(block) - file_name = imported_images_link.split("/")[-1] - item.imported_images.save(file_name, File(ntf)) + file_name, temp_file = get_archive_from_link(imported_images_link) + item.imported_images.save(file_name, File(temp_file)) return item @@ -482,6 +498,10 @@ class NewImportGroupForm(NewImportForm): def _filter_group(self, user): pass + def _need_archive(self, data): + tpe = data["importer_type"] + return [sub.importer_type.archive_required for sub in tpe.importer_types.all()] + class TargetKeyForm(forms.ModelForm): class Meta: diff --git a/ishtar_common/migrations/0230_auto_20230807_1105.py b/ishtar_common/migrations/0230_auto_20230807_1105.py deleted file mode 100644 index 9d462f393..000000000 --- a/ishtar_common/migrations/0230_auto_20230807_1105.py +++ /dev/null @@ -1,88 +0,0 @@ -# Generated by Django 2.2.24 on 2023-08-07 11:05 - -import django.core.validators -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('ishtar_common', '0229_auto_20230608_1303'), - ] - - operations = [ - migrations.CreateModel( - name='ImporterGroup', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=200, verbose_name='Name')), - ('slug', models.SlugField(max_length=100, unique=True, verbose_name='Slug')), - ('description', models.TextField(blank=True, default='', verbose_name='Description')), - ('available', models.BooleanField(default=True, verbose_name='Available')), - ], - options={ - 'verbose_name': 'Importer - Group', - 'verbose_name_plural': 'Importer - Groups', - 'ordering': ('name',), - }, - ), - migrations.AlterModelOptions( - name='import', - options={'verbose_name': 'Import - Import', 'verbose_name_plural': 'Import - Imports'}, - ), - migrations.AddField( - model_name='importertype', - name='is_import', - field=models.BooleanField(default=True, verbose_name='Can be import'), - ), - migrations.AddField( - model_name='importertype', - name='tab_number', - field=models.PositiveIntegerField(default=1, help_text='When using an Excel or Calc file choose the tab number. Keep it to 1 by default.', validators=[django.core.validators.MinValueValidator(1)], verbose_name='Tab number'), - ), - migrations.AlterField( - model_name='ishtarsiteprofile', - name='account_naming_style', - field=models.CharField(choices=[('NF', 'name.firstname'), ('FN', 'firstname.name')], default='FN', max_length=2, verbose_name='Naming style for accounts'), - ), - migrations.CreateModel( - name='ImportGroup', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=500, null=True, verbose_name='Name')), - ('imported_file', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Imported file')), - ('imported_images', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Associated images (zip file)')), - ('encoding', models.CharField(choices=[('windows-1252', 'windows-1252'), ('ISO-8859-15', 'ISO-8859-15'), ('utf-8', 'utf-8')], default='utf-8', help_text='Only required for CSV file', max_length=15, verbose_name='Encoding')), - ('csv_sep', models.CharField(choices=[(',', ','), (';', ';'), ('|', '|')], default=',', help_text='Separator for CSV file. Standard is comma but Microsoft Excel do not follow this standard and use semi-colon.', max_length=1, verbose_name='CSV separator')), - ('skip_lines', models.IntegerField(default=1, help_text='Number of header lines in your file (can be 0 and should be 0 for geopackage or Shapefile).', verbose_name='Skip lines')), - ('creation_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Creation date')), - ('end_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='End date')), - ('current_import', models.PositiveIntegerField(blank=True, null=True, verbose_name='Current import')), - ('state', models.CharField(choices=[('C', 'Created'), ('AP', 'Analyse in progress'), ('A', 'Analysed'), ('IQ', 'Import in queue'), ('IP', 'Import in progress'), ('PP', 'Post-processing in progress'), ('FE', 'Finished with errors'), ('F', 'Finished'), ('AC', 'Archived')], default='C', max_length=2, verbose_name='State')), - ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ImporterGroup', verbose_name='Importer group type')), - ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.IshtarUser')), - ], - options={ - 'verbose_name': 'Import - Group', - 'verbose_name_plural': 'Import - Groups', - }, - ), - migrations.CreateModel( - name='ImporterGroupImporter', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('order', models.PositiveIntegerField(default=10, validators=[django.core.validators.MinValueValidator(1)], verbose_name='Order')), - ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='importer_types', to='ishtar_common.ImporterGroup')), - ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='ishtar_common.ImporterType')), - ], - options={ - 'ordering': ('group', 'order'), - }, - ), - migrations.AddField( - model_name='import', - name='group', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ImportGroup', verbose_name='Group'), - ), - ] diff --git a/ishtar_common/migrations/0230_auto_20230809_1149.py b/ishtar_common/migrations/0230_auto_20230809_1149.py new file mode 100644 index 000000000..aaf97671a --- /dev/null +++ b/ishtar_common/migrations/0230_auto_20230809_1149.py @@ -0,0 +1,95 @@ +# Generated by Django 2.2.24 on 2023-08-09 11:49 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0229_auto_20230608_1303'), + ] + + operations = [ + migrations.CreateModel( + name='ImporterGroup', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=200, verbose_name='Name')), + ('slug', models.SlugField(max_length=100, unique=True, verbose_name='Slug')), + ('description', models.TextField(blank=True, default='', verbose_name='Description')), + ('available', models.BooleanField(default=True, verbose_name='Available')), + ], + options={ + 'verbose_name': 'Importer - Group', + 'verbose_name_plural': 'Importer - Groups', + 'ordering': ('name',), + }, + ), + migrations.AlterModelOptions( + name='import', + options={'verbose_name': 'Import - Import', 'verbose_name_plural': 'Import - Imports'}, + ), + migrations.AddField( + model_name='importertype', + name='archive_required', + field=models.BooleanField(default=False, verbose_name='Archive required'), + ), + migrations.AddField( + model_name='importertype', + name='is_import', + field=models.BooleanField(default=True, verbose_name='Can be import'), + ), + migrations.AddField( + model_name='importertype', + name='tab_number', + field=models.PositiveIntegerField(default=1, help_text='When using an Excel or Calc file choose the tab number. Keep it to 1 by default.', validators=[django.core.validators.MinValueValidator(1)], verbose_name='Tab number'), + ), + migrations.AlterField( + model_name='ishtarsiteprofile', + name='account_naming_style', + field=models.CharField(choices=[('NF', 'name.firstname'), ('FN', 'firstname.name')], default='FN', max_length=2, verbose_name='Naming style for accounts'), + ), + migrations.CreateModel( + name='ImportGroup', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=500, null=True, verbose_name='Name')), + ('imported_file', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Imported file')), + ('imported_images', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Associated images (zip file)')), + ('encoding', models.CharField(choices=[('windows-1252', 'windows-1252'), ('ISO-8859-15', 'ISO-8859-15'), ('utf-8', 'utf-8')], default='utf-8', help_text='Only required for CSV file', max_length=15, verbose_name='Encoding')), + ('csv_sep', models.CharField(choices=[(',', ','), (';', ';'), ('|', '|')], default=',', help_text='Separator for CSV file. Standard is comma but Microsoft Excel do not follow this standard and use semi-colon.', max_length=1, verbose_name='CSV separator')), + ('skip_lines', models.IntegerField(default=1, help_text='Number of header lines in your file (can be 0 and should be 0 for geopackage or Shapefile).', verbose_name='Skip lines')), + ('creation_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Creation date')), + ('end_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='End date')), + ('current_import', models.PositiveIntegerField(blank=True, null=True, verbose_name='Current import')), + ('state', models.CharField(choices=[('C', 'Created'), ('AP', 'Analyse in progress'), ('A', 'Analysed'), ('IQ', 'Import in queue'), ('IP', 'Import in progress'), ('PP', 'Post-processing in progress'), ('FE', 'Finished with errors'), ('F', 'Finished'), ('AC', 'Archived')], default='C', max_length=2, verbose_name='State')), + ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ImporterGroup', verbose_name='Importer group type')), + ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.IshtarUser')), + ], + options={ + 'verbose_name': 'Import - Group', + 'verbose_name_plural': 'Import - Groups', + }, + ), + migrations.AddField( + model_name='import', + name='group', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='imports', to='ishtar_common.ImportGroup', verbose_name='Group'), + ), + migrations.CreateModel( + name='ImporterGroupImporter', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('order', models.PositiveIntegerField(default=10, validators=[django.core.validators.MinValueValidator(1)], verbose_name='Order')), + ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='importer_types', to='ishtar_common.ImporterGroup')), + ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='ishtar_common.ImporterType')), + ], + options={ + 'verbose_name': 'Importer - Group <-> Importer', + 'ordering': ('group', 'order'), + 'unique_together': {('group', 'order')}, + }, + ), + ] diff --git a/ishtar_common/models.py b/ishtar_common/models.py index 607184e41..68b571af4 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -216,7 +216,6 @@ __all__ = [ "DocumentItem", "CachedGen", "StatisticItem", - "CascasdeUpdate", "Department", "State", "CompleteIdentifierItem", diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py index 76148a1a6..735ad0b10 100644 --- a/ishtar_common/models_imports.py +++ b/ishtar_common/models_imports.py @@ -184,6 +184,7 @@ class ImporterType(models.Model): ) is_template = models.BooleanField(_("Can be exported"), default=False) is_import = models.BooleanField(_("Can be import"), default=True) + archive_required = models.BooleanField(_("Archive required"), default=False) unicity_keys = models.CharField( _('Unicity keys (separator ";")'), blank=True, null=True, max_length=500, help_text=_("Mandatory for update importer. Set to key that identify items " @@ -209,7 +210,7 @@ class ImporterType(models.Model): @property def type_label(self): if self.type in IMPORT_TYPES_DICT: - return IMPORT_TYPES_DICT[self.type] + return IMPORT_TYPES_DICT[str(self.type)] return "" def get_libreoffice_template(self): @@ -416,6 +417,8 @@ class ImporterGroupImporter(models.Model): class Meta: ordering = ("group", "order") + unique_together = ("group", "order") + verbose_name = _("Importer - Group <-> Importer") def get_associated_model(parent_model, keys): @@ -1255,7 +1258,7 @@ class BaseImport(models.Model): null=True, ) imported_images = models.FileField( - _("Associated images (zip file)"), + _("Associated documents (zip file)"), upload_to="upload/imports/%Y/%m/", blank=True, null=True, @@ -1305,6 +1308,9 @@ class ImportGroup(BaseImport): verbose_name_plural = _("Import - Groups") ADMIN_SECTION = _("Imports") + def __str__(self): + return f"{self.name} ({self.importer_type.name})" + @property def status(self): if self.state not in IMPORT_GROUP_STATE_DCT: @@ -1326,7 +1332,7 @@ class ImportGroup(BaseImport): actions.append(("I", _("Re-import"))) actions.append(("AC", _("Archive"))) if self.state == "AC": - state = "FE" if self.error_file else "F" + state = "FE" if any([1 for imp in self.imports.all() if imp.error_file]) else "F" actions.append((state, _("Unarchive"))) if self.state in ("C", "A"): actions.append(("ED", _("Edit"))) @@ -1334,7 +1340,32 @@ class ImportGroup(BaseImport): return actions def save(self, *args, **kwargs): + add = self._state.adding super().save(*args, **kwargs) + if not add: + return + name = f"{self.name} ({self.importer_type.name})" + for import_type_relation in self.importer_type.importer_types.all(): + import_type = import_type_relation.importer_type + imp = Import.objects.create( + name=name, + importer_type=import_type, + group=self + ) + modified = False + # TODO: only get the relevant sheet + if self.imported_file: + imported_file = ContentFile(self.imported_file.read()) + imported_file.name = self.imported_file.name + imp.imported_file = imported_file + modified = True + if import_type.archive_required and self.imported_images: + imported_image = ContentFile(self.imported_images.read()) + imported_image.name = self.imported_images.name + imp.imported_images = imported_image + modified = True + if modified: + imp.save() class Import(BaseImport): diff --git a/ishtar_common/templates/ishtar/import_table.html b/ishtar_common/templates/ishtar/import_table.html index aec59babc..cf33341d1 100644 --- a/ishtar_common/templates/ishtar/import_table.html +++ b/ishtar_common/templates/ishtar/import_table.html @@ -99,6 +99,49 @@ $("#import-list").find('input').prop('disabled', true); {% endif %} + {% if not import.importer_type.type_label %} {# group #} + {% for sub in import.imports.all %} + + + + {{sub.importer_type}} + + + {% if sub.imported_file %} + {% trans "Source file" %} + {% endif %} + + + {% if sub.imported_images %} + {% trans "Media file" %} + {% else %} + – + {% endif %} + + + + + {{sub.status}} + + + + + {% if sub.need_matching %} + {% trans "Match"%} + {% endif %} + + {% if sub.error_file %} + {% trans "File" context "not a directory" %} + {% endif %} + {% if sub.result_file %} + {% trans "File" context "not a directory" %} + {% endif %} + {% if sub.match_file %} + {% trans "File" context "not a directory" %} + {% endif %} + + {% endfor %} + {% endif %} {% endfor %} {% endif %} diff --git a/ishtar_common/views.py b/ishtar_common/views.py index 2beade02d..2965ada87 100644 --- a/ishtar_common/views.py +++ b/ishtar_common/views.py @@ -1533,7 +1533,7 @@ class ImportListView(IshtarMixin, LoginRequiredMixin, ListView): user = models.IshtarUser.objects.get(pk=self.request.user.pk) q1 = q1.filter(user=user) q2 = q2.filter(user=user) - q1 = q1.order_by("-creation_date", "-pk") + q1 = q1.filter(group__isnull=True).order_by("-creation_date", "-pk") q2 = q2.order_by("-creation_date", "-pk") return reversed(sorted(list(q1) + list(q2), key=lambda x: x.creation_date)) -- cgit v1.2.3