Maintenance scripts: delete deprecated and migrate to ishtar_maintenance

1 files changed, 117 insertions, 18 deletions

diff --git a/ishtar_common/management/commands/ishtar_maintenance.py b/ishtar_common/management/commands/ishtar_maintenance.py
index 4f050088e..d57fb575e 100644
--- a/ishtar_common/management/commands/ishtar_maintenance.py
+++ b/ishtar_common/management/commands/ishtar_maintenance.py
@@ -1,14 +1,18 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 from argparse import RawTextHelpFormatter
 import csv
 import datetime
 import os
+import tempfile
+import shutil
 import sys
 
 from django.apps import apps
 from django.conf import settings
+from django.contrib.auth.management import create_permissions
+from django.core.exceptions import FieldDoesNotExist
 from django.core.management.base import BaseCommand, CommandError
 from django.template.defaultfilters import slugify
 
@@ -28,8 +32,12 @@ if not os.path.exists(log_path):
     os.mkdir(log_path, mode=0o770)
 
 
-def task_update_search_vectors(quiet=False, log=False):
+def task_update_search_vectors(options):
+    quiet = options.get("quiet", False)
+    log = options.get("log", False)
     for model in apps.get_models():
+        if options.get('model', None) and model.__name__ != options['model']:
+            continue
         if not hasattr(model, "update_search_vector") or \
                 not (getattr(model, "BASE_SEARCH_VECTORS", None) or
                      getattr(model, "INT_SEARCH_VECTORS", None) or
@@ -64,8 +72,12 @@ def task_update_search_vectors(quiet=False, log=False):
         _end_task(changed_nb, msg, quiet, store_results, log, log_name, csv_cols)
 
 
-def task_check_cached_label(quiet=False, log=False):
+def task_check_cached_label(options):
+    quiet = options.get("quiet", False)
+    log = options.get("log", False)
     for model in apps.get_models():
+        if options.get('model', None) and model.__name__ != options['model']:
+            continue
         if model.__name__.startswith("Historical"):
             continue
         if hasattr(model, "CACHED_LABELS") and model.CACHED_LABELS:
@@ -116,6 +128,32 @@ def task_check_cached_label(quiet=False, log=False):
         _end_task(changed_nb, msg, quiet, store_results, log, log_name, csv_cols)
 
 
+def task_update_external_id(options):
+    quiet = options.get("quiet", False)
+    for model in apps.get_models():
+        if options.get('model', None) and model.__name__ != options['model']:
+            continue
+        if model.__name__.startswith("Historical"):
+            continue
+        if not bool(
+                [k for k in dir(model) if k == "external_id"]):
+            continue
+        msg = "-> processing {}: ".format(model._meta.verbose_name)
+        ln = model.objects.count()
+        for idx, item in enumerate(model.objects.all()):
+            item.skip_history_when_saving = True
+            item.external_id = ""
+            item.complete_identifier = ""
+            item._no_move = True
+            if not quiet:
+                cmsg = "\r{} {}/{}".format(msg, idx + 1, ln)
+                sys.stdout.write(cmsg)
+                sys.stdout.flush()
+            item.save()
+        if not quiet:
+            sys.stdout.write("\n")
+
+
 def _end_task(changed_nb, msg, quiet, store_results, log, log_name, csv_cols):
     if not quiet:
         if changed_nb:
@@ -134,7 +172,8 @@ def _end_task(changed_nb, msg, quiet, store_results, log, log_name, csv_cols):
             sys.stdout.write(f"log: {path} written.\n")
 
 
-def task_main_image(quiet=False, log=False):
+def task_main_image(options):
+    quiet = options.get("quiet", False)
     for model in apps.get_models():
         if not issubclass(model, models_common.DocumentItem):
             continue
@@ -155,14 +194,44 @@ def task_main_image(quiet=False, log=False):
             sys.stdout.write(f"{nb} main image fixed for {model.__name__}\n")
 
 
-def task_missing_parcels(quiet=False, log=False):
+def task_regenerate_qrcodes(options):
+    quiet = options.get("quiet", False)
+    secure = not options['no-https']
+    for model in apps.get_models():
+        if options.get('model', None) and model.__name__ != options['model']:
+            continue
+        try:
+            model._meta.get_field('qrcode')
+        except FieldDoesNotExist:
+            continue
+        msg = "-> processing {}: ".format(model._meta.verbose_name)
+        ln = model.objects.count()
+        tmpdir = tempfile.mkdtemp("-qrcode")
+        for idx, obj in enumerate(model.objects.all()):
+            obj.skip_history_when_saving = True
+            obj._no_move = True
+            if not quiet:
+                cmsg = "\r{} {}/{}".format(msg, idx + 1, ln)
+                sys.stdout.write(cmsg)
+                sys.stdout.flush()
+            obj.generate_qrcode(secure=secure, tmpdir=tmpdir)
+        shutil.rmtree(tmpdir)
+        if not quiet:
+            sys.stdout.write("\n")
+
+
+def task_regenerate_permissions(options):
+    for app in ['ishtar_common', 'archaeological_operations',
+                'archaeological_context_records',
+                'archaeological_finds', 'archaeological_warehouse']:
+        create_permissions(apps.get_app_config(app))
+
+
+def task_missing_parcels(options):
+    quiet = options.get("quiet", False)
     Parcel = apps.get_model("archaeological_operations", "Parcel")
     q = Parcel.objects.filter(context_record__isnull=False, operation=None)
     nb = q.count()
-    if not nb:
-        if not quiet:
-            sys.stdout.write("No parcel to fix.\n")
-            return
     for idx, parcel in enumerate(q.all()):
         if not quiet:
             sys.stdout.write(f"\r[{percent(idx, nb)}] {idx + 1}/{nb}")
@@ -175,6 +244,13 @@ def task_missing_parcels(quiet=False, log=False):
         parcel.save()
     if not quiet:
         sys.stdout.write("\n")
+    if not options.get("clean", False):
+        return
+    q = Parcel.objects.filter(associated_file=None, operation=None)
+    nb = q.count()
+    q.delete()
+    if not quiet:
+        sys.stdout.write(f"{nb} orphan parcel deleted.\n")
 
 
 def percent(current, total):
@@ -186,21 +262,34 @@ def get_time():
 
 
 TASKS = {
-    "main_image": {
+    "fix_main_image": {
         "help": "for items with images and no main image, put the first one created as a main image",
         "action": task_main_image,
     },
-    "cached_label": {
-        "help": "regenerate cached label on all tables",
+    "fix_missing_parcels": {
+        "help": "fix lost parcel association on operation from context records. "
+                "With --clean option delete orphan parcels.",
+        "action": task_missing_parcels,
+    },
+    "update_cached_label": {
+        "help": "regenerate cached label",
         "action": task_check_cached_label,
     },
+    "update_permissions": {
+        "help": "regenerate basic model permissions",
+        "action": task_regenerate_permissions,
+    },
+    "update_qrcodes": {
+        "help": "regenerate qrcodes",
+        "action": task_regenerate_qrcodes,
+    },
     "update_search_vector": {
-        "help": "regenerate search vectors on all tables",
+        "help": "regenerate search vectors",
         "action": task_update_search_vectors,
     },
-    "operation_missing_parcels": {
-        "help": "fix lost parcel association on operation from context records.",
-        "action": task_missing_parcels,
+    "update_external_ids": {
+        "help": "regenerate external and complete ID",
+        "action": task_update_external_id,
     },
 }
 
@@ -226,22 +315,32 @@ class Command(BaseCommand):
         )
         parser.add_argument("task", help=task_help)
         parser.add_argument(
+            '--model', type=str, default='', dest='model',
+            help='Specific model to update')
+        parser.add_argument(
             "--quiet", dest="quiet", action="store_true", help="Quiet output"
         )
         parser.add_argument(
             "--log", dest="log", action="store_true", help="Log into a file"
         )
+        parser.add_argument(
+            '--no-https', dest='no-https', action='store_true',
+            default=False,
+            help="[update_qrcodes] Use this parameter if https is not available")
+        parser.add_argument(
+            '--clean', dest='clean', action='store_true',
+            default=False,
+            help="[operation_missing_parcels] Delete orphan parcel after fix")
 
     def handle(self, *args, **options):
         if options["task"] not in TASKS.keys():
             msg = f"{options['task']} is not a valid task. Available tasks are:\n"
             msg += "\n".join(TASKS.keys())
             raise CommandError(msg)
-        log = options["log"]
         quiet = options["quiet"]
         if not quiet:
             sys.stdout.write(f"[{get_time()}] Processing task {options['task']}\n")
-        errors = TASKS[options["task"]]["action"](quiet=quiet, log=log)
+        errors = TASKS[options["task"]]["action"](options)
         if not errors:
             if not quiet:
                 sys.stdout.write(f"[{get_time()}] Task {options['task']} finished\n")