From 37a23f20320eea2d6b85fa9bb326846aea68d7ed Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Mon, 11 Sep 2023 15:16:06 +0200 Subject: ✨ Pre-import form: modify models - admin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ishtar_common/admin.py | 6 + .../migrations/0230_auto_20230810_1140.py | 105 ---------------- .../migrations/0230_auto_20230912_1832.py | 133 +++++++++++++++++++++ ishtar_common/models.py | 6 + ishtar_common/models_imports.py | 71 ++++++++++- locale/fr/LC_MESSAGES/django.po | 8 +- 6 files changed, 216 insertions(+), 113 deletions(-) delete mode 100644 ishtar_common/migrations/0230_auto_20230810_1140.py create mode 100644 ishtar_common/migrations/0230_auto_20230912_1832.py diff --git a/ishtar_common/admin.py b/ishtar_common/admin.py index 3d1909275..af11be06c 100644 --- a/ishtar_common/admin.py +++ b/ishtar_common/admin.py @@ -1965,6 +1965,12 @@ class ItemKeyAdmin(admin.ModelAdmin): admin_site.register(models.ItemKey, ItemKeyAdmin) +@admin.register(models.ImportColumnValue, site=admin_site) +class ImportColumnValue(admin.ModelAdmin): + list_display = ("import_item", "column") + + + class JsonContentTypeFormMixin(object): class Meta: model = models.JsonDataSection diff --git a/ishtar_common/migrations/0230_auto_20230810_1140.py b/ishtar_common/migrations/0230_auto_20230810_1140.py deleted file mode 100644 index b668baa84..000000000 --- a/ishtar_common/migrations/0230_auto_20230810_1140.py +++ /dev/null @@ -1,105 +0,0 @@ -# Generated by Django 2.2.24 on 2023-08-10 11:40 - -import django.core.validators -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('ishtar_common', '0229_auto_20230608_1303'), - ] - - operations = [ - migrations.CreateModel( - name='ImporterGroup', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=200, verbose_name='Name')), - ('slug', models.SlugField(max_length=100, unique=True, verbose_name='Slug')), - ('description', models.TextField(blank=True, default='', verbose_name='Description')), - ('available', models.BooleanField(default=True, verbose_name='Available')), - ], - options={ - 'verbose_name': 'Importer - Group', - 'verbose_name_plural': 'Importer - Groups', - 'ordering': ('name',), - }, - ), - migrations.AlterModelOptions( - name='import', - options={'verbose_name': 'Import - Import', 'verbose_name_plural': 'Import - Imports'}, - ), - migrations.AddField( - model_name='import', - name='next_import', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='imports', to='ishtar_common.Import', verbose_name='Next import'), - ), - migrations.AddField( - model_name='importertype', - name='archive_required', - field=models.BooleanField(default=False, verbose_name='Archive required'), - ), - migrations.AddField( - model_name='importertype', - name='is_import', - field=models.BooleanField(default=True, verbose_name='Can be import'), - ), - migrations.AddField( - model_name='importertype', - name='tab_number', - field=models.PositiveIntegerField(default=1, help_text='When using an Excel or Calc file choose the tab number. Keep it to 1 by default.', validators=[django.core.validators.MinValueValidator(1)], verbose_name='Tab number'), - ), - migrations.AlterField( - model_name='import', - name='imported_images', - field=models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Associated documents (zip file)'), - ), - migrations.AlterField( - model_name='ishtarsiteprofile', - name='account_naming_style', - field=models.CharField(choices=[('NF', 'name.firstname'), ('FN', 'firstname.name')], default='FN', max_length=2, verbose_name='Naming style for accounts'), - ), - migrations.CreateModel( - name='ImportGroup', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=500, null=True, verbose_name='Name')), - ('imported_file', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Imported file')), - ('imported_images', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Associated documents (zip file)')), - ('encoding', models.CharField(choices=[('windows-1252', 'windows-1252'), ('ISO-8859-15', 'ISO-8859-15'), ('utf-8', 'utf-8')], default='utf-8', help_text='Only required for CSV file', max_length=15, verbose_name='Encoding')), - ('csv_sep', models.CharField(choices=[(',', ','), (';', ';'), ('|', '|')], default=',', help_text='Separator for CSV file. Standard is comma but Microsoft Excel do not follow this standard and use semi-colon.', max_length=1, verbose_name='CSV separator')), - ('skip_lines', models.IntegerField(default=1, help_text='Number of header lines in your file (can be 0 and should be 0 for geopackage or Shapefile).', verbose_name='Skip lines')), - ('creation_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Creation date')), - ('end_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='End date')), - ('current_import', models.PositiveIntegerField(blank=True, null=True, verbose_name='Current import')), - ('state', models.CharField(choices=[('C', 'Created'), ('AP', 'Analyse in progress'), ('A', 'Analysed'), ('IQ', 'Import in queue'), ('IP', 'Import in progress'), ('PP', 'Post-processing in progress'), ('FE', 'Finished with errors'), ('F', 'Finished'), ('AC', 'Archived')], default='C', max_length=2, verbose_name='State')), - ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ImporterGroup', verbose_name='Importer group type')), - ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.IshtarUser')), - ], - options={ - 'verbose_name': 'Import - Group', - 'verbose_name_plural': 'Import - Groups', - }, - ), - migrations.AddField( - model_name='import', - name='group', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='imports', to='ishtar_common.ImportGroup', verbose_name='Group'), - ), - migrations.CreateModel( - name='ImporterGroupImporter', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('order', models.PositiveIntegerField(default=10, validators=[django.core.validators.MinValueValidator(1)], verbose_name='Order')), - ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='importer_types', to='ishtar_common.ImporterGroup')), - ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='ishtar_common.ImporterType')), - ], - options={ - 'verbose_name': 'Importer - Group <-> Importer', - 'ordering': ('group', 'order'), - 'unique_together': {('group', 'order')}, - }, - ), - ] diff --git a/ishtar_common/migrations/0230_auto_20230912_1832.py b/ishtar_common/migrations/0230_auto_20230912_1832.py new file mode 100644 index 000000000..649e0dccb --- /dev/null +++ b/ishtar_common/migrations/0230_auto_20230912_1832.py @@ -0,0 +1,133 @@ +# Generated by Django 2.2.24 on 2023-09-12 18:32 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0229_auto_20230608_1303'), + ] + + operations = [ + migrations.CreateModel( + name='ImporterGroup', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=200, verbose_name='Name')), + ('slug', models.SlugField(max_length=100, unique=True, verbose_name='Slug')), + ('description', models.TextField(blank=True, default='', verbose_name='Description')), + ('available', models.BooleanField(default=True, verbose_name='Available')), + ], + options={ + 'verbose_name': 'Importer - Group', + 'verbose_name_plural': 'Importer - Groups', + 'ordering': ('name',), + }, + ), + migrations.AlterModelOptions( + name='import', + options={'verbose_name': 'Import - Import', 'verbose_name_plural': 'Import - Imports'}, + ), + migrations.AlterModelOptions( + name='itemkey', + options={'verbose_name': 'Import - Item key', 'verbose_name_plural': 'Import - Item keys'}, + ), + migrations.AddField( + model_name='import', + name='next_import', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='imports', to='ishtar_common.Import', verbose_name='Next import'), + ), + migrations.AddField( + model_name='importertype', + name='archive_required', + field=models.BooleanField(default=False, verbose_name='Archive required'), + ), + migrations.AddField( + model_name='importertype', + name='is_import', + field=models.BooleanField(default=True, verbose_name='Can be import'), + ), + migrations.AddField( + model_name='importertype', + name='pre_import_message', + field=models.TextField(blank=True, default='', max_length=500, verbose_name='Pre-import form message'), + ), + migrations.AddField( + model_name='importertype', + name='tab_number', + field=models.PositiveIntegerField(default=1, help_text='When using an Excel or Calc file choose the tab number. Keep it to 1 by default.', validators=[django.core.validators.MinValueValidator(1)], verbose_name='Tab number'), + ), + migrations.AlterField( + model_name='import', + name='imported_images', + field=models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Associated documents (zip file)'), + ), + migrations.AlterField( + model_name='importercolumn', + name='col_number', + field=models.SmallIntegerField(default=1, help_text='Column number in the table. Put 0 or negative number for pre-importer field.', verbose_name='Column number'), + ), + migrations.AlterField( + model_name='ishtarsiteprofile', + name='account_naming_style', + field=models.CharField(choices=[('NF', 'name.firstname'), ('FN', 'firstname.name')], default='FN', max_length=2, verbose_name='Naming style for accounts'), + ), + migrations.CreateModel( + name='ImportGroup', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=500, null=True, verbose_name='Name')), + ('imported_file', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Imported file')), + ('imported_images', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Associated documents (zip file)')), + ('encoding', models.CharField(choices=[('windows-1252', 'windows-1252'), ('ISO-8859-15', 'ISO-8859-15'), ('utf-8', 'utf-8')], default='utf-8', help_text='Only required for CSV file', max_length=15, verbose_name='Encoding')), + ('csv_sep', models.CharField(choices=[(',', ','), (';', ';'), ('|', '|')], default=',', help_text='Separator for CSV file. Standard is comma but Microsoft Excel do not follow this standard and use semi-colon.', max_length=1, verbose_name='CSV separator')), + ('skip_lines', models.IntegerField(default=1, help_text='Number of header lines in your file (can be 0 and should be 0 for geopackage or Shapefile).', verbose_name='Skip lines')), + ('creation_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Creation date')), + ('end_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='End date')), + ('current_import', models.PositiveIntegerField(blank=True, null=True, verbose_name='Current import')), + ('state', models.CharField(choices=[('C', 'Created'), ('AP', 'Analyse in progress'), ('A', 'Analysed'), ('IQ', 'Import in queue'), ('IP', 'Import in progress'), ('PP', 'Post-processing in progress'), ('FE', 'Finished with errors'), ('F', 'Finished'), ('AC', 'Archived')], default='C', max_length=2, verbose_name='State')), + ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ImporterGroup', verbose_name='Importer group type')), + ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.IshtarUser')), + ], + options={ + 'verbose_name': 'Import - Group', + 'verbose_name_plural': 'Import - Groups', + }, + ), + migrations.AddField( + model_name='import', + name='group', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='imports', to='ishtar_common.ImportGroup', verbose_name='Group'), + ), + migrations.CreateModel( + name='ImporterGroupImporter', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('order', models.PositiveIntegerField(default=10, validators=[django.core.validators.MinValueValidator(1)], verbose_name='Order')), + ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='importer_types', to='ishtar_common.ImporterGroup')), + ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='ishtar_common.ImporterType')), + ], + options={ + 'verbose_name': 'Importer - Group <-> Importer', + 'ordering': ('group', 'order'), + 'unique_together': {('group', 'order')}, + }, + ), + migrations.CreateModel( + name='ImportColumnValue', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('value', models.TextField(blank=True, default='')), + ('column', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ImporterColumn')), + ('import_item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.Import')), + ], + options={ + 'verbose_name': 'Import - Pre-import value', + 'verbose_name_plural': 'Import - Pre-import values', + 'unique_together': {('column', 'import_item')}, + }, + ), + ] diff --git a/ishtar_common/models.py b/ishtar_common/models.py index 68b571af4..a38fc3c3e 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -124,6 +124,7 @@ from ishtar_common.models_imports import ( TargetKeyGroup, ValueFormater, ItemKey, + ImportColumnValue, ) from ishtar_common.utils import ( @@ -175,6 +176,7 @@ from ishtar_common.models_common import ( ) __all__ = [ + "ImportColumnValue", "ImporterModel", "ImporterType", "ImporterGroup", @@ -3466,6 +3468,10 @@ class IshtarUser(FullSearch): def class_verbose_name(cls): return cls._meta.verbose_name + @property + def is_superuser(self) -> bool: + return self.user_ptr.is_superuser + def show_field_number(self, update=False): cache_key, value = get_cache(self.__class__, ["show_field_number"]) if not update and value is not None: diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py index 3fbd7653d..454bed7e8 100644 --- a/ishtar_common/models_imports.py +++ b/ishtar_common/models_imports.py @@ -195,6 +195,9 @@ class ImporterType(models.Model): "1 key.") ) available = models.BooleanField(_("Available"), default=True) + pre_import_message = models.TextField( + _("Pre-import form message"), blank=True, default="", max_length=500 + ) objects = SlugModelManager() SERIALIZATION_EXCLUDE = ["users"] @@ -630,7 +633,10 @@ class ImporterColumn(models.Model): importer_type = models.ForeignKey( ImporterType, related_name="columns", on_delete=models.CASCADE ) - col_number = models.IntegerField(_("Column number"), default=1) + col_number = models.SmallIntegerField( + _("Column number"), default=1, + help_text=_("Column number in the table. Put 0 or negative number for pre-importer field.") + ) description = models.TextField(_("Description"), blank=True, null=True) regexp_pre_filter = models.ForeignKey( "Regexp", @@ -1080,6 +1086,21 @@ DATE_FORMATS = ( IMPORTER_TYPES_CHOICES = {"TypeFormater": TARGET_MODELS, "DateFormater": DATE_FORMATS} +FORMATER_WIDGETS_DCT = { + "IntegerFormater": ("IntegerField", None), + "FloatFormater": ("FloatField", None), + "UnicodeFormater": ("CharField", None), + "DateFormater": ("DateField", None), + "TypeFormater": ("ChoiceField", None), + "YearFormater": ("IntegerField", None), + "StrToBoolean": ("BooleanField", None), + "FileFormater": ("FileField", None), + "InseeFormater": ("IntegerField", None), + "UpperFormater": ("CharField", None), + "LowerFormater": ("CharField", None), + "UnknowType": ("CharField", None), +} + class FormaterTypeManager(models.Manager): def get_by_natural_key(self, formater_type, options, many_split): @@ -1090,7 +1111,7 @@ class FormaterTypeManager(models.Manager): class FormaterType(models.Model): formater_type = models.CharField( - "Formater type", max_length=20, choices=IMPORTER_TYPES + _("Formater type"), max_length=20, choices=IMPORTER_TYPES ) options = models.CharField(_("Options"), max_length=500, blank=True, null=True) many_split = models.CharField( @@ -1327,6 +1348,10 @@ class ImportGroup(BaseImport): def __str__(self): return f"{self.name} ({self.importer_type.name})" + @property + def has_pre_import_form(self) -> bool: + return False + @property def import_id(self): return f"group-{self.id}" @@ -1547,6 +1572,11 @@ class Import(BaseImport): def __str__(self): return "{} | {}".format(self.name or "-", self.importer_type) + def is_available(self, ishtar_user) -> bool: + if ishtar_user.is_superuser or self.user == ishtar_user: + return True + return False + @property def import_id(self): return str(self.id) @@ -1569,6 +1599,21 @@ class Import(BaseImport): errors.append(row) return errors + @property + def has_pre_import_form(self) -> bool: + """ + Check if a pre-import form is available + """ + return bool(self.importer_type.columns.filter(col_number__lte=0).count()) + + @property + def pre_import_form_is_valid(self) -> bool: + for column in self.importer_type.columns.filter(col_number__lte=0, required=True): + q = ImportColumnValue.objects.filter(column=column, import_item=self) + if not q.count() or q.all()[0].value == "": + return False + return True + def get_number_of_lines(self): if self.number_of_line: return self.number_of_line @@ -2186,6 +2231,24 @@ def pre_delete_import(sender, **kwargs): pre_delete.connect(pre_delete_import, sender=Import) +class ImportColumnValue(models.Model): + """ + Value in a column for pre-import columns + """ + column = models.ForeignKey(ImporterColumn, on_delete=models.CASCADE) + import_item = models.ForeignKey(Import, on_delete=models.CASCADE) + value = models.TextField(default="", blank=True) + + class Meta: + verbose_name = _("Import - Pre-import value") + verbose_name_plural = _("Import - Pre-import values") + unique_together = ("column", "import_item") + ADMIN_SECTION = _("Imports") + + def __str__(self): + return f"{self.import_item} - {self.column} - {self.value}" + + class ItemKey(models.Model): key = models.TextField(_("Key")) content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) @@ -2206,8 +2269,8 @@ class ItemKey(models.Model): ) class Meta: - verbose_name = _("Importer - Item key") - verbose_name_plural = _("Imports - Item keys") + verbose_name = _("Import - Item key") + verbose_name_plural = _("Import - Item keys") ADMIN_SECTION = _("Imports") def __str__(self): diff --git a/locale/fr/LC_MESSAGES/django.po b/locale/fr/LC_MESSAGES/django.po index 97f108744..bd53a62b9 100644 --- a/locale/fr/LC_MESSAGES/django.po +++ b/locale/fr/LC_MESSAGES/django.po @@ -12165,12 +12165,12 @@ msgid "Specific key to an import" msgstr "Clé spécifique à un import" #: ishtar_common/models_imports.py:1969 -msgid "Importer - Item key" -msgstr "Importeur - Clé de rapprochement d'élément" +msgid "Import - Item key" +msgstr "Import - Clé de rapprochement" #: ishtar_common/models_imports.py:1970 -msgid "Imports - Item keys" -msgstr "Importeur - Clés de rapprochement d'éléments" +msgid "Import - Item keys" +msgstr "Import - Clés de rapprochement" #: ishtar_common/models_rest.py:49 ishtar_common/models_rest.py:63 msgid "User" -- cgit v1.2.3