diff options
| -rw-r--r-- | ishtar_common/admin.py | 19 | ||||
| -rw-r--r-- | ishtar_common/migrations/0230_auto_20230729_1345.py | 78 | ||||
| -rw-r--r-- | ishtar_common/models.py | 4 | ||||
| -rw-r--r-- | ishtar_common/models_imports.py | 117 |
4 files changed, 192 insertions, 26 deletions
diff --git a/ishtar_common/admin.py b/ishtar_common/admin.py index 764dd2624..2b706f10a 100644 --- a/ishtar_common/admin.py +++ b/ishtar_common/admin.py @@ -1666,6 +1666,25 @@ class ImporterTypeAdmin(ImportJSONActionAdmin): prepopulated_fields = {"slug": ("name",)} +class ImporterGroupImporterInline(admin.TabularInline): + model = models.ImporterGroupImporter + extra = 3 + + +@admin.register(models.ImporterGroup, site=admin_site) +class ImporterTypeAdmin(admin.ModelAdmin): + list_display = ("name", "importer_types_label", "available") + actions = [ + change_value("available", True, _("Make available")), + change_value("available", False, _("Make unavailable")), + ] + list_filter = ["available"] + search_fields = ["name"] + prepopulated_fields = {"slug": ("name",)} + inlines = [ImporterGroupImporterInline] + + + class RegexpAdmin(admin.ModelAdmin): list_display = ("name", "regexp", "description") diff --git a/ishtar_common/migrations/0230_auto_20230729_1345.py b/ishtar_common/migrations/0230_auto_20230729_1345.py new file mode 100644 index 000000000..8d09fe16c --- /dev/null +++ b/ishtar_common/migrations/0230_auto_20230729_1345.py @@ -0,0 +1,78 @@ +# Generated by Django 2.2.24 on 2023-07-29 13:45 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0229_auto_20230608_1303'), + ] + + operations = [ + migrations.CreateModel( + name='ImporterGroup', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=200, verbose_name='Name')), + ('slug', models.SlugField(max_length=100, unique=True, verbose_name='Slug')), + ('description', models.TextField(blank=True, default='', verbose_name='Description')), + ('available', models.BooleanField(default=True, verbose_name='Available')), + ], + options={ + 'verbose_name': 'Importer - Group', + 'verbose_name_plural': 'Importer - Groups', + 'ordering': ('name',), + }, + ), + migrations.AddField( + model_name='importertype', + name='is_import', + field=models.BooleanField(default=False, verbose_name='Can be import'), + ), + migrations.AddField( + model_name='importertype', + name='tab_number', + field=models.PositiveIntegerField(default=1, help_text='When using an Excel or Calc file choose the tab number. Keep it to 1 by default.', validators=[django.core.validators.MinValueValidator(1)], verbose_name='Tab number'), + ), + migrations.AlterField( + model_name='ishtarsiteprofile', + name='account_naming_style', + field=models.CharField(choices=[('NF', 'name.firstname'), ('FN', 'firstname.name')], default='FN', max_length=2, verbose_name='Naming style for accounts'), + ), + migrations.CreateModel( + name='ImportGroup', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=500, null=True, verbose_name='Name')), + ('imported_file', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Imported file')), + ('imported_images', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Associated images (zip file)')), + ('encoding', models.CharField(choices=[('windows-1252', 'windows-1252'), ('ISO-8859-15', 'ISO-8859-15'), ('utf-8', 'utf-8')], default='utf-8', help_text='Only required for CSV file', max_length=15, verbose_name='Encoding')), + ('csv_sep', models.CharField(choices=[(',', ','), (';', ';'), ('|', '|')], default=',', help_text='Separator for CSV file. Standard is comma but Microsoft Excel do not follow this standard and use semi-colon.', max_length=1, verbose_name='CSV separator')), + ('skip_lines', models.IntegerField(default=1, help_text='Number of header lines in your file (can be 0 and should be 0 for geopackage or Shapefile).', verbose_name='Skip lines')), + ('creation_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Creation date')), + ('end_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='End date')), + ('current_import', models.PositiveIntegerField(blank=True, null=True, verbose_name='Current import')), + ('state', models.CharField(choices=[('C', 'Created'), ('AP', 'Analyse in progress'), ('A', 'Analysed'), ('IQ', 'Import in queue'), ('IP', 'Import in progress'), ('PP', 'Post-processing in progress'), ('FE', 'Finished with errors'), ('F', 'Finished'), ('AC', 'Archived')], default='C', max_length=2, verbose_name='State')), + ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ImporterGroup', verbose_name='Importer group type')), + ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.IshtarUser')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='ImporterGroupImporter', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('order', models.PositiveIntegerField(default=10, validators=[django.core.validators.MinValueValidator(1)], verbose_name='Order')), + ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='importer_types', to='ishtar_common.ImporterGroup')), + ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='ishtar_common.ImporterType')), + ], + options={ + 'ordering': ('group', 'order'), + }, + ), + ] diff --git a/ishtar_common/models.py b/ishtar_common/models.py index b011e541f..6cc31216c 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -109,6 +109,8 @@ from ishtar_common.model_merging import merge_model_objects from ishtar_common.models_imports import ( ImporterModel, ImporterType, + ImporterGroup, + ImporterGroupImporter, ImporterDefault, ImporterDefaultValues, ImporterColumn, @@ -174,6 +176,8 @@ from ishtar_common.models_common import ( __all__ = [ "ImporterModel", "ImporterType", + "ImporterGroup", + "ImporterGroupImporter", "ImporterDefault", "ImporterDefaultValues", "ImporterColumn", diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py index e91a94868..457d3ff4e 100644 --- a/ishtar_common/models_imports.py +++ b/ishtar_common/models_imports.py @@ -42,7 +42,7 @@ from django.contrib.gis.geos.error import GEOSException from django.core.exceptions import ValidationError, SuspiciousOperation from django.core.files import File from django.core.files.base import ContentFile -from django.core.validators import validate_comma_separated_integer_list +from django.core.validators import validate_comma_separated_integer_list, MinValueValidator from django.db.models.base import ModelBase from django.db.models.signals import pre_delete from django.template.defaultfilters import slugify @@ -129,11 +129,6 @@ class ImporterModel(models.Model): return (self.klass,) -class ImporterTypeManager(models.Manager): - def get_by_natural_key(self, slug): - return self.get(slug=slug) - - IMPORT_TYPES = ( ("tab", _("Table")), ("gis", _("GIS")), @@ -155,6 +150,10 @@ class ImporterType(models.Model): type = models.CharField( _("Type"), max_length=3, choices=IMPORT_TYPES, default="tab" ) + tab_number = models.PositiveIntegerField( + _("Tab number"), default=1, validators=[MinValueValidator(1)], + help_text=_("When using an Excel or Calc file choose the tab number. Keep it to 1 by default.") + ) layer_name = models.CharField( _("Layer name"), max_length=200, @@ -184,6 +183,7 @@ class ImporterType(models.Model): related_name="importer_type_created", ) is_template = models.BooleanField(_("Can be exported"), default=False) + is_import = models.BooleanField(_("Can be import"), default=False) unicity_keys = models.CharField( _('Unicity keys (separator ";")'), blank=True, null=True, max_length=500, help_text=_("Mandatory for update importer. Set to key that identify items " @@ -191,7 +191,7 @@ class ImporterType(models.Model): "1 key.") ) available = models.BooleanField(_("Available"), default=True) - objects = ImporterTypeManager() + objects = SlugModelManager() SERIALIZATION_EXCLUDE = ["users"] class Meta: @@ -377,13 +377,47 @@ class ImporterType(models.Model): col_names.append(formater.label) return cols, col_names - def save(self, *args, **kwargs): if not self.slug: self.slug = create_slug(ImporterType, self.name) return super(ImporterType, self).save(*args, **kwargs) +class ImporterGroup(models.Model): + name = models.CharField(_("Name"), max_length=200) + slug = models.SlugField(_("Slug"), unique=True, max_length=100) + description = models.TextField( + _("Description"), blank=True, default="" + ) + available = models.BooleanField(_("Available"), default=True) + + class Meta: + verbose_name = _("Importer - Group") + verbose_name_plural = _("Importer - Groups") + ordering = ("name",) + ADMIN_SECTION = _("Imports") + objects = SlugModelManager() + + def natural_key(self): + return self.slug, + + def __str__(self): + return self.name + + @property + def importer_types_label(self) -> str: + return " ; ".join([imp.importer_type.name for imp in self.importer_types.all()]) + + +class ImporterGroupImporter(models.Model): + group = models.ForeignKey(ImporterGroup, on_delete=models.CASCADE, related_name="importer_types") + importer_type = models.ForeignKey(ImporterType, on_delete=models.CASCADE, related_name="groups") + order = models.PositiveIntegerField(_("Order"), default=10, validators=[MinValueValidator(1)]) + + class Meta: + ordering = ("group", "order") + + def get_associated_model(parent_model, keys): model = None if isinstance(parent_model, str): @@ -1133,6 +1167,21 @@ IMPORT_STATE = ( ) IMPORT_STATE_DCT = dict(IMPORT_STATE) + +IMPORT_GROUP_STATE = ( + ("C", _("Created")), + ("AP", _("Analyse in progress")), + ("A", _("Analysed")), + ("IQ", _("Import in queue")), + ("IP", _("Import in progress")), + ("PP", _("Post-processing in progress")), + ("FE", _("Finished with errors")), + ("F", _("Finished")), + ("AC", _("Archived")), +) + +IMPORT_GROUP_STATE_DCT = dict(IMPORT_STATE) + ENCODINGS = [ (settings.ENCODING, settings.ENCODING), (settings.ALT_ENCODING, settings.ALT_ENCODING), @@ -1192,13 +1241,11 @@ IMPORT_GEOMETRY = { } -class Import(models.Model): +class BaseImport(models.Model): user = models.ForeignKey( "IshtarUser", blank=True, null=True, on_delete=models.SET_NULL ) name = models.CharField(_("Name"), max_length=500, null=True) - importer_type = models.ForeignKey(ImporterType, on_delete=models.CASCADE, - verbose_name=_("Importer type")) imported_file = models.FileField( _("Imported file"), upload_to="upload/imports/%Y/%m/", @@ -1215,15 +1262,6 @@ class Import(models.Model): max_length=220, help_text=max_size_help(), ) - associated_group = models.ForeignKey( - TargetKeyGroup, - blank=True, - null=True, - on_delete=models.SET_NULL, - help_text=_( - "If a group is selected, target key saved in this group " "will be used." - ), - ) encoding = models.CharField( _("Encoding"), choices=ENCODINGS, default="utf-8", max_length=15, help_text=_("Only required for CSV file"), @@ -1243,6 +1281,39 @@ class Import(models.Model): default=1, help_text=_("Number of header lines in your file (can be 0 and should be 0 for geopackage or Shapefile)."), ) + creation_date = models.DateTimeField( + _("Creation date"), auto_now_add=True, blank=True, null=True + ) + end_date = models.DateTimeField( + _("End date"), auto_now_add=True, blank=True, null=True, editable=False + ) + + class Meta: + abstract = True + + +class ImportGroup(BaseImport): + importer_type = models.ForeignKey(ImporterGroup, on_delete=models.CASCADE, + verbose_name=_("Importer group type")) + current_import = models.PositiveIntegerField(_("Current import"), blank=True, null=True) + state = models.CharField( + _("State"), max_length=2, choices=IMPORT_GROUP_STATE, default="C" + ) + + +class Import(BaseImport): + importer_type = models.ForeignKey(ImporterType, on_delete=models.CASCADE, + verbose_name=_("Importer type")) + # TODO - associated_group: relevant? + associated_group = models.ForeignKey( + TargetKeyGroup, + blank=True, + null=True, + on_delete=models.SET_NULL, + help_text=_( + "If a group is selected, target key saved in this group will be used." + ), + ) error_file = models.FileField( _("Error file"), upload_to="upload/imports/%Y/%m/", @@ -1283,12 +1354,6 @@ class Import(models.Model): default=False, help_text=_("If set to true, do not overload existing values."), ) - creation_date = models.DateTimeField( - _("Creation date"), auto_now_add=True, blank=True, null=True - ) - end_date = models.DateTimeField( - _("End date"), auto_now_add=True, blank=True, null=True, editable=False - ) seconds_remaining = models.IntegerField( _("Remaining seconds"), blank=True, null=True, editable=False ) |