✨ imports - key matches: use unicode on keys

4 files changed, 105 insertions, 9 deletions

diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index ffd1cf6ac..8e0671cd8 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -35,7 +35,7 @@ from django.core.exceptions import FieldDoesNotExist, FieldError, MultipleObject
 from django.core.files import File
 from django.db import IntegrityError, DatabaseError, transaction
 from django.db.models import Q
-from django.template.defaultfilters import slugify
+from django.utils.text import slugify
 from django.utils.translation import gettext_lazy as _
 
 from ishtar_common.utils import (
@@ -606,7 +606,7 @@ class TypeFormater(StrChoiceFormater):
         return slug
 
     def prepare(self, value):
-        return slugify(str(value).strip())
+        return slugify(str(value).strip(), allow_unicode=True)
 
     def add_key(self, obj, value, ishtar_import=None):
         obj.add_key(slugify(value), force=True, ishtar_import=ishtar_import)
@@ -734,7 +734,7 @@ class StrToBoolean(Formater, ChoiceChecker):
     def prepare(self, value):
         value = str(value).strip()
         if not self.strict:
-            value = slugify(value)
+            value = slugify(value, allow_unicode=True)
         return value
 
     def check(
diff --git a/ishtar_common/management/commands/process_initialize_item_keys.py b/ishtar_common/management/commands/process_initialize_item_keys.py
new file mode 100644
index 000000000..4ccaadc0a
--- /dev/null
+++ b/ishtar_common/management/commands/process_initialize_item_keys.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import datetime
+import sys
+
+from django.apps import apps
+from django.conf import settings
+from django.contrib.contenttypes.models import ContentType
+from django.core.management.base import BaseCommand
+from django.utils.text import slugify
+
+from ishtar_common.models import ItemKey
+from ishtar_common.utils import get_log_time, get_percent, get_eta, BColors
+
+
+def write_output(base_lbl, idx, total, ref_time):
+    lbl = f"\r{BColors.OKBLUE}[{get_percent(idx, total)}] {base_lbl} {idx + 1}/{total}"
+    lbl += f" ({get_eta(idx, total, ref_time, datetime.datetime.now())} left){BColors.ENDC}"
+    sys.stdout.write(lbl)
+    sys.stdout.flush()
+
+
+def migrate_item_key(clean_old=False):
+    """
+    clean_old=False: set to True to migrate from non unicode to unicode
+    """
+    SPACE = " " * 30
+    for app in ("ishtar_common", "archaeological_context_records", "archaeological_files",
+                "archaeological_finds", "archaeological_operations",
+                "archaeological_warehouse"):
+        app_models = apps.get_app_config(app).get_models()
+        for model in app_models:
+            if any(1 for attr in ("available", "txt_idx", "comment", "available")
+                   if not hasattr(model, attr)):
+                continue  # not a general type
+            content_type = ContentType.objects.get(app_label=app,
+                                                   model=model._meta.model_name)
+            ref_time = datetime.datetime.now()
+            q = model.objects
+            nb = q.count()
+            for idx, item in enumerate(q.all()):
+                write_output(model._meta.verbose_name, idx, nb, ref_time)
+                if clean_old:
+                    ItemKey.objects.filter(
+                        key=slugify(item.label),
+                        content_type=content_type, importer_type=None,
+                        ishtar_import=None, user=None, group=None).delete()
+                ItemKey.objects.get_or_create(
+                    key=slugify(item.label, allow_unicode=True),
+                    content_type=content_type, object_id=item.pk,
+                    importer_type=None, ishtar_import=None, user=None,
+                    group=None)
+            lbl = f"\r{BColors.OKGREEN}* {model._meta.verbose_name} - OK{SPACE}{BColors.ENDC}\n"
+            sys.stdout.write(lbl)
+
+
+class Command(BaseCommand):
+    help = "Re-initialize ItemKey"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--quiet", dest="quiet", action="store_true", help="Quiet output"
+        )
+        parser.add_argument(
+            "--clean-old", dest="clean_old", action="store_true",
+            help="V4.4 migration - clean old non-utf-8 keys"
+        )
+
+    def handle(self, *args, **options):
+        global quiet
+        quiet = options["quiet"]
+        if not quiet:
+            sys.stdout.write(f"{BColors.OKGREEN}[{get_log_time()}] Processing{BColors.ENDC}")
+        settings.USE_BACKGROUND_TASK = False
+        migrate_item_key(clean_old=options["clean_old"])
+        sys.exit(1)
diff --git a/ishtar_common/migrations/0267_reinitialize_item_keys.py b/ishtar_common/migrations/0267_reinitialize_item_keys.py
new file mode 100644
index 000000000..53696c164
--- /dev/null
+++ b/ishtar_common/migrations/0267_reinitialize_item_keys.py
@@ -0,0 +1,17 @@
+from django.db import migrations
+from django.core.management import call_command
+
+
+def load_data(apps, __):
+    call_command("process_initialize_item_keys", "--quiet", "--clean-old")
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ishtar_common', '0266_migrate_cached_town'),
+    ]
+
+    operations = [
+        migrations.RunPython(load_data)
+    ]
diff --git a/ishtar_common/models_common.py b/ishtar_common/models_common.py
index 02e47ad56..c90ca7309 100644
--- a/ishtar_common/models_common.py
+++ b/ishtar_common/models_common.py
@@ -40,11 +40,11 @@ from django.urls import reverse, NoReverseMatch
 from django.core.validators import validate_slug
 from django.db import connection, transaction, OperationalError, IntegrityError
 from django.db.models import JSONField, Q, Count, Max
-from django.db.models.signals import m2m_changed, post_save, post_delete, pre_delete
+from django.db.models.signals import m2m_changed, post_save, post_delete
 from django.template import loader
-from django.template.defaultfilters import slugify
 from django.utils import timezone
 from django.utils.safestring import SafeText, mark_safe
+from django.utils.text import slugify
 from django.utils.translation import activate, deactivate
 from ishtar_common.utils import (
     BColors,
@@ -684,16 +684,18 @@ class GeneralType(Cached, models.Model):
         if self.pk:
             old = self.__class__.objects.get(pk=self.pk)
             content_type = ContentType.objects.get_for_model(self.__class__)
-            if slugify(self.label) != slugify(old.label):
+            if slugify(self.label, allow_unicode=True) != slugify(old.label,
+                                                                  allow_unicode=True):
                 ItemKey.objects.filter(
-                    object_id=self.pk, key=slugify(old.label), content_type=content_type
+                    object_id=self.pk, key=slugify(old.label, allow_unicode=True),
+                    content_type=content_type
                 ).delete()
             if self.txt_idx != old.txt_idx:
                 ItemKey.objects.filter(
                     object_id=self.pk, key=old.txt_idx, content_type=content_type
                 ).delete()
 
-        obj = super(GeneralType, self).save(*args, **kwargs)
+        obj = super().save(*args, **kwargs)
         self.generate_key(force=True)
         return obj
 
@@ -725,7 +727,7 @@ class GeneralType(Cached, models.Model):
         ItemKey.objects.get_or_create(**filtr)
 
     def generate_key(self, force=False):
-        for key in (slugify(self.label), self.txt_idx):
+        for key in (slugify(self.label, allow_unicode=True), self.txt_idx):
             self.add_key(key)
 
     def get_keys(self, current_import):