✨ Imports groups: models, admin

4 files changed, 192 insertions, 26 deletions

diff --git a/ishtar_common/admin.py b/ishtar_common/admin.py
index 764dd2624..2b706f10a 100644
--- a/ishtar_common/admin.py
+++ b/ishtar_common/admin.py
@@ -1666,6 +1666,25 @@ class ImporterTypeAdmin(ImportJSONActionAdmin):
     prepopulated_fields = {"slug": ("name",)}
 
 
+class ImporterGroupImporterInline(admin.TabularInline):
+    model = models.ImporterGroupImporter
+    extra = 3
+
+
+@admin.register(models.ImporterGroup, site=admin_site)
+class ImporterTypeAdmin(admin.ModelAdmin):
+    list_display = ("name", "importer_types_label", "available")
+    actions = [
+        change_value("available", True, _("Make available")),
+        change_value("available", False, _("Make unavailable")),
+    ]
+    list_filter = ["available"]
+    search_fields = ["name"]
+    prepopulated_fields = {"slug": ("name",)}
+    inlines = [ImporterGroupImporterInline]
+
+
+
 class RegexpAdmin(admin.ModelAdmin):
     list_display = ("name", "regexp", "description")
 
diff --git a/ishtar_common/migrations/0230_auto_20230729_1345.py b/ishtar_common/migrations/0230_auto_20230729_1345.py
new file mode 100644
index 000000000..8d09fe16c
--- /dev/null
+++ b/ishtar_common/migrations/0230_auto_20230729_1345.py
@@ -0,0 +1,78 @@
+# Generated by Django 2.2.24 on 2023-07-29 13:45
+
+import django.core.validators
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ishtar_common', '0229_auto_20230608_1303'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ImporterGroup',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=200, verbose_name='Name')),
+                ('slug', models.SlugField(max_length=100, unique=True, verbose_name='Slug')),
+                ('description', models.TextField(blank=True, default='', verbose_name='Description')),
+                ('available', models.BooleanField(default=True, verbose_name='Available')),
+            ],
+            options={
+                'verbose_name': 'Importer - Group',
+                'verbose_name_plural': 'Importer - Groups',
+                'ordering': ('name',),
+            },
+        ),
+        migrations.AddField(
+            model_name='importertype',
+            name='is_import',
+            field=models.BooleanField(default=False, verbose_name='Can be import'),
+        ),
+        migrations.AddField(
+            model_name='importertype',
+            name='tab_number',
+            field=models.PositiveIntegerField(default=1, help_text='When using an Excel or Calc file choose the tab number. Keep it to 1 by default.', validators=[django.core.validators.MinValueValidator(1)], verbose_name='Tab number'),
+        ),
+        migrations.AlterField(
+            model_name='ishtarsiteprofile',
+            name='account_naming_style',
+            field=models.CharField(choices=[('NF', 'name.firstname'), ('FN', 'firstname.name')], default='FN', max_length=2, verbose_name='Naming style for accounts'),
+        ),
+        migrations.CreateModel(
+            name='ImportGroup',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=500, null=True, verbose_name='Name')),
+                ('imported_file', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Imported file')),
+                ('imported_images', models.FileField(blank=True, help_text='La taille maximale supportée pour le fichier est de 100 Mo.', max_length=220, null=True, upload_to='upload/imports/%Y/%m/', verbose_name='Associated images (zip file)')),
+                ('encoding', models.CharField(choices=[('windows-1252', 'windows-1252'), ('ISO-8859-15', 'ISO-8859-15'), ('utf-8', 'utf-8')], default='utf-8', help_text='Only required for CSV file', max_length=15, verbose_name='Encoding')),
+                ('csv_sep', models.CharField(choices=[(',', ','), (';', ';'), ('|', '|')], default=',', help_text='Separator for CSV file. Standard is comma but Microsoft Excel do not follow this standard and use semi-colon.', max_length=1, verbose_name='CSV separator')),
+                ('skip_lines', models.IntegerField(default=1, help_text='Number of header lines in your file (can be 0 and should be 0 for geopackage or Shapefile).', verbose_name='Skip lines')),
+                ('creation_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Creation date')),
+                ('end_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='End date')),
+                ('current_import', models.PositiveIntegerField(blank=True, null=True, verbose_name='Current import')),
+                ('state', models.CharField(choices=[('C', 'Created'), ('AP', 'Analyse in progress'), ('A', 'Analysed'), ('IQ', 'Import in queue'), ('IP', 'Import in progress'), ('PP', 'Post-processing in progress'), ('FE', 'Finished with errors'), ('F', 'Finished'), ('AC', 'Archived')], default='C', max_length=2, verbose_name='State')),
+                ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ImporterGroup', verbose_name='Importer group type')),
+                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.IshtarUser')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='ImporterGroupImporter',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('order', models.PositiveIntegerField(default=10, validators=[django.core.validators.MinValueValidator(1)], verbose_name='Order')),
+                ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='importer_types', to='ishtar_common.ImporterGroup')),
+                ('importer_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='ishtar_common.ImporterType')),
+            ],
+            options={
+                'ordering': ('group', 'order'),
+            },
+        ),
+    ]
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index b011e541f..6cc31216c 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -109,6 +109,8 @@ from ishtar_common.model_merging import merge_model_objects
 from ishtar_common.models_imports import (
     ImporterModel,
     ImporterType,
+    ImporterGroup,
+    ImporterGroupImporter,
     ImporterDefault,
     ImporterDefaultValues,
     ImporterColumn,
@@ -174,6 +176,8 @@ from ishtar_common.models_common import (
 __all__ = [
     "ImporterModel",
     "ImporterType",
+    "ImporterGroup",
+    "ImporterGroupImporter",
     "ImporterDefault",
     "ImporterDefaultValues",
     "ImporterColumn",
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index e91a94868..457d3ff4e 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -42,7 +42,7 @@ from django.contrib.gis.geos.error import GEOSException
 from django.core.exceptions import ValidationError, SuspiciousOperation
 from django.core.files import File
 from django.core.files.base import ContentFile
-from django.core.validators import validate_comma_separated_integer_list
+from django.core.validators import validate_comma_separated_integer_list, MinValueValidator
 from django.db.models.base import ModelBase
 from django.db.models.signals import pre_delete
 from django.template.defaultfilters import slugify
@@ -129,11 +129,6 @@ class ImporterModel(models.Model):
         return (self.klass,)
 
 
-class ImporterTypeManager(models.Manager):
-    def get_by_natural_key(self, slug):
-        return self.get(slug=slug)
-
-
 IMPORT_TYPES = (
     ("tab", _("Table")),
     ("gis", _("GIS")),
@@ -155,6 +150,10 @@ class ImporterType(models.Model):
     type = models.CharField(
         _("Type"), max_length=3, choices=IMPORT_TYPES, default="tab"
     )
+    tab_number = models.PositiveIntegerField(
+        _("Tab number"), default=1, validators=[MinValueValidator(1)],
+        help_text=_("When using an Excel or Calc file choose the tab number. Keep it to 1 by default.")
+    )
     layer_name = models.CharField(
         _("Layer name"),
         max_length=200,
@@ -184,6 +183,7 @@ class ImporterType(models.Model):
         related_name="importer_type_created",
     )
     is_template = models.BooleanField(_("Can be exported"), default=False)
+    is_import = models.BooleanField(_("Can be import"), default=False)
     unicity_keys = models.CharField(
         _('Unicity keys (separator ";")'), blank=True, null=True, max_length=500,
         help_text=_("Mandatory for update importer. Set to key that identify items "
@@ -191,7 +191,7 @@ class ImporterType(models.Model):
                     "1 key.")
     )
     available = models.BooleanField(_("Available"), default=True)
-    objects = ImporterTypeManager()
+    objects = SlugModelManager()
     SERIALIZATION_EXCLUDE = ["users"]
 
     class Meta:
@@ -377,13 +377,47 @@ class ImporterType(models.Model):
             col_names.append(formater.label)
         return cols, col_names
 
-
     def save(self, *args, **kwargs):
         if not self.slug:
             self.slug = create_slug(ImporterType, self.name)
         return super(ImporterType, self).save(*args, **kwargs)
 
 
+class ImporterGroup(models.Model):
+    name = models.CharField(_("Name"), max_length=200)
+    slug = models.SlugField(_("Slug"), unique=True, max_length=100)
+    description = models.TextField(
+        _("Description"), blank=True, default=""
+    )
+    available = models.BooleanField(_("Available"), default=True)
+
+    class Meta:
+        verbose_name = _("Importer - Group")
+        verbose_name_plural = _("Importer - Groups")
+        ordering = ("name",)
+    ADMIN_SECTION = _("Imports")
+    objects = SlugModelManager()
+
+    def natural_key(self):
+        return self.slug,
+
+    def __str__(self):
+        return self.name
+
+    @property
+    def importer_types_label(self) -> str:
+        return " ; ".join([imp.importer_type.name for imp in self.importer_types.all()])
+
+
+class ImporterGroupImporter(models.Model):
+    group = models.ForeignKey(ImporterGroup, on_delete=models.CASCADE, related_name="importer_types")
+    importer_type = models.ForeignKey(ImporterType, on_delete=models.CASCADE, related_name="groups")
+    order = models.PositiveIntegerField(_("Order"), default=10, validators=[MinValueValidator(1)])
+
+    class Meta:
+        ordering = ("group", "order")
+
+
 def get_associated_model(parent_model, keys):
     model = None
     if isinstance(parent_model, str):
@@ -1133,6 +1167,21 @@ IMPORT_STATE = (
 )
 
 IMPORT_STATE_DCT = dict(IMPORT_STATE)
+
+IMPORT_GROUP_STATE = (
+    ("C", _("Created")),
+    ("AP", _("Analyse in progress")),
+    ("A", _("Analysed")),
+    ("IQ", _("Import in queue")),
+    ("IP", _("Import in progress")),
+    ("PP", _("Post-processing in progress")),
+    ("FE", _("Finished with errors")),
+    ("F", _("Finished")),
+    ("AC", _("Archived")),
+)
+
+IMPORT_GROUP_STATE_DCT = dict(IMPORT_STATE)
+
 ENCODINGS = [
     (settings.ENCODING, settings.ENCODING),
     (settings.ALT_ENCODING, settings.ALT_ENCODING),
@@ -1192,13 +1241,11 @@ IMPORT_GEOMETRY = {
 }
 
 
-class Import(models.Model):
+class BaseImport(models.Model):
     user = models.ForeignKey(
         "IshtarUser", blank=True, null=True, on_delete=models.SET_NULL
     )
     name = models.CharField(_("Name"), max_length=500, null=True)
-    importer_type = models.ForeignKey(ImporterType, on_delete=models.CASCADE,
-                                      verbose_name=_("Importer type"))
     imported_file = models.FileField(
         _("Imported file"),
         upload_to="upload/imports/%Y/%m/",
@@ -1215,15 +1262,6 @@ class Import(models.Model):
         max_length=220,
         help_text=max_size_help(),
     )
-    associated_group = models.ForeignKey(
-        TargetKeyGroup,
-        blank=True,
-        null=True,
-        on_delete=models.SET_NULL,
-        help_text=_(
-            "If a group is selected, target key saved in this group " "will be used."
-        ),
-    )
     encoding = models.CharField(
         _("Encoding"), choices=ENCODINGS, default="utf-8", max_length=15,
         help_text=_("Only required for CSV file"),
@@ -1243,6 +1281,39 @@ class Import(models.Model):
         default=1,
         help_text=_("Number of header lines in your file (can be 0 and should be 0 for geopackage or Shapefile)."),
     )
+    creation_date = models.DateTimeField(
+        _("Creation date"), auto_now_add=True, blank=True, null=True
+    )
+    end_date = models.DateTimeField(
+        _("End date"), auto_now_add=True, blank=True, null=True, editable=False
+    )
+
+    class Meta:
+        abstract = True
+
+
+class ImportGroup(BaseImport):
+    importer_type = models.ForeignKey(ImporterGroup, on_delete=models.CASCADE,
+                                      verbose_name=_("Importer group type"))
+    current_import = models.PositiveIntegerField(_("Current import"), blank=True, null=True)
+    state = models.CharField(
+        _("State"), max_length=2, choices=IMPORT_GROUP_STATE, default="C"
+    )
+
+
+class Import(BaseImport):
+    importer_type = models.ForeignKey(ImporterType, on_delete=models.CASCADE,
+                                      verbose_name=_("Importer type"))
+    # TODO - associated_group: relevant?
+    associated_group = models.ForeignKey(
+        TargetKeyGroup,
+        blank=True,
+        null=True,
+        on_delete=models.SET_NULL,
+        help_text=_(
+            "If a group is selected, target key saved in this group will be used."
+        ),
+    )
     error_file = models.FileField(
         _("Error file"),
         upload_to="upload/imports/%Y/%m/",
@@ -1283,12 +1354,6 @@ class Import(models.Model):
         default=False,
         help_text=_("If set to true, do not overload existing values."),
     )
-    creation_date = models.DateTimeField(
-        _("Creation date"), auto_now_add=True, blank=True, null=True
-    )
-    end_date = models.DateTimeField(
-        _("End date"), auto_now_add=True, blank=True, null=True, editable=False
-    )
     seconds_remaining = models.IntegerField(
         _("Remaining seconds"), blank=True, null=True, editable=False
     )