diff options
| -rw-r--r-- | archaeological_operations/tests.py | 2 | ||||
| -rw-r--r-- | ishtar_common/data_importer.py | 6 | ||||
| -rw-r--r-- | ishtar_common/management/commands/process_initialize_item_keys.py | 77 | ||||
| -rw-r--r-- | ishtar_common/migrations/0267_reinitialize_item_keys.py | 17 | ||||
| -rw-r--r-- | ishtar_common/models_common.py | 12 |
5 files changed, 105 insertions, 9 deletions
diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py index fd7173a46..5661e8505 100644 --- a/archaeological_operations/tests.py +++ b/archaeological_operations/tests.py @@ -254,7 +254,7 @@ class ImportTest(BaseImportTest): # target for another user username, password, user = create_user() another_user = IshtarUser.objects.get(pk=user.pk) - tgs = list(TargetKey.objects.filter(key="neolithik").all()) + tgs = list(TargetKey.objects.filter(key="néolithik").all()) for tg in tgs[1:]: tg.delete() target4 = tgs[0] diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index 80392218d..5bb3f428d 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -36,7 +36,7 @@ from django.core.exceptions import FieldError, MultipleObjectsReturned from django.core.files import File from django.db import IntegrityError, DatabaseError, transaction from django.db.models import Q -from django.template.defaultfilters import slugify +from django.utils.text import slugify from django.utils.translation import ugettext_lazy as _ from ishtar_common.utils import ( @@ -586,7 +586,7 @@ class TypeFormater(StrChoiceFormater): self.equiv_dict[key] = item def prepare(self, value): - return slugify(str(value).strip()) + return slugify(str(value).strip(), allow_unicode=True) def add_key(self, obj, value, ishtar_import=None): obj.add_key(slugify(value), force=True, ishtar_import=ishtar_import) @@ -714,7 +714,7 @@ class StrToBoolean(Formater, ChoiceChecker): def prepare(self, value): value = str(value).strip() if not self.strict: - value = slugify(value) + value = slugify(value, allow_unicode=True) return value def check( diff --git a/ishtar_common/management/commands/process_initialize_item_keys.py b/ishtar_common/management/commands/process_initialize_item_keys.py new file mode 100644 index 000000000..4ccaadc0a --- /dev/null +++ b/ishtar_common/management/commands/process_initialize_item_keys.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import datetime +import sys + +from django.apps import apps +from django.conf import settings +from django.contrib.contenttypes.models import ContentType +from django.core.management.base import BaseCommand +from django.utils.text import slugify + +from ishtar_common.models import ItemKey +from ishtar_common.utils import get_log_time, get_percent, get_eta, BColors + + +def write_output(base_lbl, idx, total, ref_time): + lbl = f"\r{BColors.OKBLUE}[{get_percent(idx, total)}] {base_lbl} {idx + 1}/{total}" + lbl += f" ({get_eta(idx, total, ref_time, datetime.datetime.now())} left){BColors.ENDC}" + sys.stdout.write(lbl) + sys.stdout.flush() + + +def migrate_item_key(clean_old=False): + """ + clean_old=False: set to True to migrate from non unicode to unicode + """ + SPACE = " " * 30 + for app in ("ishtar_common", "archaeological_context_records", "archaeological_files", + "archaeological_finds", "archaeological_operations", + "archaeological_warehouse"): + app_models = apps.get_app_config(app).get_models() + for model in app_models: + if any(1 for attr in ("available", "txt_idx", "comment", "available") + if not hasattr(model, attr)): + continue # not a general type + content_type = ContentType.objects.get(app_label=app, + model=model._meta.model_name) + ref_time = datetime.datetime.now() + q = model.objects + nb = q.count() + for idx, item in enumerate(q.all()): + write_output(model._meta.verbose_name, idx, nb, ref_time) + if clean_old: + ItemKey.objects.filter( + key=slugify(item.label), + content_type=content_type, importer_type=None, + ishtar_import=None, user=None, group=None).delete() + ItemKey.objects.get_or_create( + key=slugify(item.label, allow_unicode=True), + content_type=content_type, object_id=item.pk, + importer_type=None, ishtar_import=None, user=None, + group=None) + lbl = f"\r{BColors.OKGREEN}* {model._meta.verbose_name} - OK{SPACE}{BColors.ENDC}\n" + sys.stdout.write(lbl) + + +class Command(BaseCommand): + help = "Re-initialize ItemKey" + + def add_arguments(self, parser): + parser.add_argument( + "--quiet", dest="quiet", action="store_true", help="Quiet output" + ) + parser.add_argument( + "--clean-old", dest="clean_old", action="store_true", + help="V4.4 migration - clean old non-utf-8 keys" + ) + + def handle(self, *args, **options): + global quiet + quiet = options["quiet"] + if not quiet: + sys.stdout.write(f"{BColors.OKGREEN}[{get_log_time()}] Processing{BColors.ENDC}") + settings.USE_BACKGROUND_TASK = False + migrate_item_key(clean_old=options["clean_old"]) + sys.exit(1) diff --git a/ishtar_common/migrations/0267_reinitialize_item_keys.py b/ishtar_common/migrations/0267_reinitialize_item_keys.py new file mode 100644 index 000000000..53696c164 --- /dev/null +++ b/ishtar_common/migrations/0267_reinitialize_item_keys.py @@ -0,0 +1,17 @@ +from django.db import migrations +from django.core.management import call_command + + +def load_data(apps, __): + call_command("process_initialize_item_keys", "--quiet", "--clean-old") + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0266_migrate_cached_town'), + ] + + operations = [ + migrations.RunPython(load_data) + ] diff --git a/ishtar_common/models_common.py b/ishtar_common/models_common.py index 10c564024..71efb0305 100644 --- a/ishtar_common/models_common.py +++ b/ishtar_common/models_common.py @@ -43,8 +43,8 @@ from django.db import connection, transaction, OperationalError, IntegrityError from django.db.models import Q, Count, Max from django.db.models.signals import post_save, post_delete, m2m_changed from django.template import loader -from django.template.defaultfilters import slugify from django.utils.safestring import SafeText, mark_safe +from django.utils.text import slugify from django.utils.translation import activate, deactivate from ishtar_common.utils import ( BColors, @@ -684,16 +684,18 @@ class GeneralType(Cached, models.Model): if self.pk: old = self.__class__.objects.get(pk=self.pk) content_type = ContentType.objects.get_for_model(self.__class__) - if slugify(self.label) != slugify(old.label): + if slugify(self.label, allow_unicode=True) != slugify(old.label, + allow_unicode=True): ItemKey.objects.filter( - object_id=self.pk, key=slugify(old.label), content_type=content_type + object_id=self.pk, key=slugify(old.label, allow_unicode=True), + content_type=content_type ).delete() if self.txt_idx != old.txt_idx: ItemKey.objects.filter( object_id=self.pk, key=old.txt_idx, content_type=content_type ).delete() - obj = super(GeneralType, self).save(*args, **kwargs) + obj = super().save(*args, **kwargs) self.generate_key(force=True) return obj @@ -725,7 +727,7 @@ class GeneralType(Cached, models.Model): ItemKey.objects.get_or_create(**filtr) def generate_key(self, force=False): - for key in (slugify(self.label), self.txt_idx): + for key in (slugify(self.label, allow_unicode=True), self.txt_idx): self.add_key(key) def get_keys(self, current_import): |