From 0dc167611a2f9fecc58b9709487362ceb3eb9752 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Thu, 2 Mar 2023 16:14:45 +0100 Subject: Maintenance scripts: delete deprecated and migrate to ishtar_maintenance --- CHANGES.md | 1 + ishtar_common/management/commands/clean_ishtar.py | 34 --- .../commands/fix_missing_cached_labels.py | 95 --------- .../management/commands/generate_external_ids.py | 41 ---- .../commands/generate_merge_candidates.py | 41 ---- .../management/commands/ishtar_maintenance.py | 135 ++++++++++-- .../management/commands/ishtar_migrate_odts.py | 86 -------- .../management/commands/ishtar_migrate_srid.py | 62 ------ ishtar_common/management/commands/load_towns.py | 22 -- .../commands/migrate_relations_cache_tables.py | 59 ++++++ ishtar_common/management/commands/migrate_srid.py | 62 ++++++ .../commands/process_generate_merge_candidates.py | 41 ++++ .../commands/process_reassociate_similar_images.py | 222 ++++++++++++++++++++ .../commands/reassociate_similar_images.py | 232 --------------------- .../management/commands/regenerate_external_id.py | 71 ------- .../management/commands/regenerate_permissions.py | 35 ---- .../management/commands/regenerate_qrcodes.py | 78 ------- .../regenerate_search_vector_cached_label.py | 77 ------- .../commands/relations_update_cache_tables.py | 59 ------ .../management/commands/update_external_ids.py | 26 --- .../management/commands/update_search_vectors.py | 48 ----- .../commands/update_specific_importers.py | 31 --- ishtar_common/models.py | 4 +- ishtar_common/tasks.py | 33 --- locale/fr/LC_MESSAGES/django.po | 4 +- 25 files changed, 506 insertions(+), 1093 deletions(-) delete mode 100644 ishtar_common/management/commands/clean_ishtar.py delete mode 100644 ishtar_common/management/commands/fix_missing_cached_labels.py delete mode 100644 ishtar_common/management/commands/generate_external_ids.py delete mode 100644 ishtar_common/management/commands/generate_merge_candidates.py delete mode 100644 ishtar_common/management/commands/ishtar_migrate_odts.py delete mode 100644 ishtar_common/management/commands/ishtar_migrate_srid.py delete mode 100644 ishtar_common/management/commands/load_towns.py create mode 100644 ishtar_common/management/commands/migrate_relations_cache_tables.py create mode 100644 ishtar_common/management/commands/migrate_srid.py create mode 100644 ishtar_common/management/commands/process_generate_merge_candidates.py create mode 100644 ishtar_common/management/commands/process_reassociate_similar_images.py delete mode 100644 ishtar_common/management/commands/reassociate_similar_images.py delete mode 100644 ishtar_common/management/commands/regenerate_external_id.py delete mode 100644 ishtar_common/management/commands/regenerate_permissions.py delete mode 100644 ishtar_common/management/commands/regenerate_qrcodes.py delete mode 100644 ishtar_common/management/commands/regenerate_search_vector_cached_label.py delete mode 100644 ishtar_common/management/commands/relations_update_cache_tables.py delete mode 100644 ishtar_common/management/commands/update_external_ids.py delete mode 100644 ishtar_common/management/commands/update_search_vectors.py delete mode 100644 ishtar_common/management/commands/update_specific_importers.py diff --git a/CHANGES.md b/CHANGES.md index 57626d74e..7ba5306bc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -26,6 +26,7 @@ Ishtar changelog - Free text search: - use accent and unaccented string - add french_archeo thesaurus config +- Maintenance scripts: delete deprecated and migrate to ishtar_maintenance ### Bug fixes ### - Search: diff --git a/ishtar_common/management/commands/clean_ishtar.py b/ishtar_common/management/commands/clean_ishtar.py deleted file mode 100644 index 3117022ee..000000000 --- a/ishtar_common/management/commands/clean_ishtar.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (C) 2017 Étienne Loks - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -# See the file COPYING for details. - -import sys - -from django.core.management.base import BaseCommand - -from archaeological_operations.models import Parcel - - -class Command(BaseCommand): - args = '' - help = 'Clean unused items' - - def handle(self, *args, **options): - for parcel in Parcel.objects.all(): - parcel.clean_orphan() - self.stdout.write('Parcel cleaned.\n') diff --git a/ishtar_common/management/commands/fix_missing_cached_labels.py b/ishtar_common/management/commands/fix_missing_cached_labels.py deleted file mode 100644 index ad2767c56..000000000 --- a/ishtar_common/management/commands/fix_missing_cached_labels.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (C) 2013-2018 Étienne Loks - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -# See the file COPYING for details. - -import sys - -from django.core.management.base import BaseCommand - -from django.apps import apps -from django.db.models import Q - - -APPS = ['ishtar_common', 'archaeological_operations', - 'archaeological_context_records', 'archaeological_finds', - 'archaeological_warehouse'] - - -class Command(BaseCommand): - args = '' - help = 'Regenerate cached labels and search vectors' - - def add_arguments(self, parser): - parser.add_argument('app_name', nargs='?', default=None, - choices=APPS) - parser.add_argument('model_name', nargs='?', default=None) - parser.add_argument( - '--quiet', dest='quiet', action='store_true', - help='Quiet output') - - def handle(self, *args, **options): - quiet = options['quiet'] - limit = options['app_name'] - model_name = options['model_name'] - if model_name: - model_name = model_name.lower() - for app in APPS: - if limit and app != limit: - continue - if not quiet: - print("* app: {}".format(app)) - for model in apps.get_app_config(app).get_models(): - if model_name and model.__name__.lower() != model_name: - continue - if model.__name__.startswith('Historical'): - continue - if not bool( - [k for k in dir(model) - if k.startswith('_generate_') or - k == "search_vector"]): - continue - if hasattr(model, "CACHED_LABELS") and model.CACHED_LABELS: - cached_keys = model.CACHED_LABELS - elif hasattr(model, "cached_label") \ - and "Basket" not in model.__name__: - cached_keys = ["cached_label"] - else: - continue - query = None - for cached in cached_keys: - subquery = Q(**{cached + "__isnull": True}) - subquery |= Q(**{cached: ""}) - if not query: - query = subquery - else: - query |= subquery - q = model.objects.filter(query) - msg = "-> processing {}: ".format(model._meta.verbose_name) - ln = q.count() - for idx, obj_id in enumerate(q.values('pk').all()): - obj = model.objects.get(pk=obj_id['pk']) - obj.skip_history_when_saving = True - obj._no_move = True - obj._no_geo_check = True - cmsg = "\r{} {}/{}".format(msg, idx + 1, ln) - if not quiet: - sys.stdout.write(cmsg) - sys.stdout.flush() - obj.save() - if not quiet and ln: - sys.stdout.write("\n") diff --git a/ishtar_common/management/commands/generate_external_ids.py b/ishtar_common/management/commands/generate_external_ids.py deleted file mode 100644 index d5d3d3bed..000000000 --- a/ishtar_common/management/commands/generate_external_ids.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (C) 2017 Étienne Loks - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -# See the file COPYING for details. - -import sys - -from django.core.management.base import BaseCommand - -from archaeological_files.models import File -from archaeological_operations.models import Parcel -from archaeological_context_records.models import ContextRecord -from archaeological_finds.models import BaseFind, Find -from archaeological_warehouse.models import Warehouse, Container - - -class Command(BaseCommand): - args = '' - help = 'Regenerate external ids' - - def handle(self, *args, **options): - for model in (File, Parcel, ContextRecord, BaseFind, Find, Warehouse, - Container): - self.stdout.write('{} regenerate.\n'.format(model)) - for obj in model.objects.all(): - obj.skip_history_when_saving = True - obj.save() diff --git a/ishtar_common/management/commands/generate_merge_candidates.py b/ishtar_common/management/commands/generate_merge_candidates.py deleted file mode 100644 index 106ce8e13..000000000 --- a/ishtar_common/management/commands/generate_merge_candidates.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (C) 2014 Étienne Loks - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -# See the file COPYING for details. - -import sys - -from django.core.management.base import BaseCommand, CommandError -from django.core.exceptions import ObjectDoesNotExist - -import ishtar_common.models as models - - -class Command(BaseCommand): - args = '' - help = 'Regenerate merge candidates' - - def handle(self, *args, **options): - for model in [models.Person, models.Organization]: - sys.stdout.write('\n* %s treatment\n' % str(model)) - q = model.objects - total = q.count() - for idx, item in enumerate(q.all()): - sys.stdout.write('\r\t %d/%d' % (idx, total)) - sys.stdout.flush() - item.generate_merge_candidate() - sys.stdout.write('\nSuccessfully generation of merge candidates\n') diff --git a/ishtar_common/management/commands/ishtar_maintenance.py b/ishtar_common/management/commands/ishtar_maintenance.py index 4f050088e..d57fb575e 100644 --- a/ishtar_common/management/commands/ishtar_maintenance.py +++ b/ishtar_common/management/commands/ishtar_maintenance.py @@ -1,14 +1,18 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- from argparse import RawTextHelpFormatter import csv import datetime import os +import tempfile +import shutil import sys from django.apps import apps from django.conf import settings +from django.contrib.auth.management import create_permissions +from django.core.exceptions import FieldDoesNotExist from django.core.management.base import BaseCommand, CommandError from django.template.defaultfilters import slugify @@ -28,8 +32,12 @@ if not os.path.exists(log_path): os.mkdir(log_path, mode=0o770) -def task_update_search_vectors(quiet=False, log=False): +def task_update_search_vectors(options): + quiet = options.get("quiet", False) + log = options.get("log", False) for model in apps.get_models(): + if options.get('model', None) and model.__name__ != options['model']: + continue if not hasattr(model, "update_search_vector") or \ not (getattr(model, "BASE_SEARCH_VECTORS", None) or getattr(model, "INT_SEARCH_VECTORS", None) or @@ -64,8 +72,12 @@ def task_update_search_vectors(quiet=False, log=False): _end_task(changed_nb, msg, quiet, store_results, log, log_name, csv_cols) -def task_check_cached_label(quiet=False, log=False): +def task_check_cached_label(options): + quiet = options.get("quiet", False) + log = options.get("log", False) for model in apps.get_models(): + if options.get('model', None) and model.__name__ != options['model']: + continue if model.__name__.startswith("Historical"): continue if hasattr(model, "CACHED_LABELS") and model.CACHED_LABELS: @@ -116,6 +128,32 @@ def task_check_cached_label(quiet=False, log=False): _end_task(changed_nb, msg, quiet, store_results, log, log_name, csv_cols) +def task_update_external_id(options): + quiet = options.get("quiet", False) + for model in apps.get_models(): + if options.get('model', None) and model.__name__ != options['model']: + continue + if model.__name__.startswith("Historical"): + continue + if not bool( + [k for k in dir(model) if k == "external_id"]): + continue + msg = "-> processing {}: ".format(model._meta.verbose_name) + ln = model.objects.count() + for idx, item in enumerate(model.objects.all()): + item.skip_history_when_saving = True + item.external_id = "" + item.complete_identifier = "" + item._no_move = True + if not quiet: + cmsg = "\r{} {}/{}".format(msg, idx + 1, ln) + sys.stdout.write(cmsg) + sys.stdout.flush() + item.save() + if not quiet: + sys.stdout.write("\n") + + def _end_task(changed_nb, msg, quiet, store_results, log, log_name, csv_cols): if not quiet: if changed_nb: @@ -134,7 +172,8 @@ def _end_task(changed_nb, msg, quiet, store_results, log, log_name, csv_cols): sys.stdout.write(f"log: {path} written.\n") -def task_main_image(quiet=False, log=False): +def task_main_image(options): + quiet = options.get("quiet", False) for model in apps.get_models(): if not issubclass(model, models_common.DocumentItem): continue @@ -155,14 +194,44 @@ def task_main_image(quiet=False, log=False): sys.stdout.write(f"{nb} main image fixed for {model.__name__}\n") -def task_missing_parcels(quiet=False, log=False): +def task_regenerate_qrcodes(options): + quiet = options.get("quiet", False) + secure = not options['no-https'] + for model in apps.get_models(): + if options.get('model', None) and model.__name__ != options['model']: + continue + try: + model._meta.get_field('qrcode') + except FieldDoesNotExist: + continue + msg = "-> processing {}: ".format(model._meta.verbose_name) + ln = model.objects.count() + tmpdir = tempfile.mkdtemp("-qrcode") + for idx, obj in enumerate(model.objects.all()): + obj.skip_history_when_saving = True + obj._no_move = True + if not quiet: + cmsg = "\r{} {}/{}".format(msg, idx + 1, ln) + sys.stdout.write(cmsg) + sys.stdout.flush() + obj.generate_qrcode(secure=secure, tmpdir=tmpdir) + shutil.rmtree(tmpdir) + if not quiet: + sys.stdout.write("\n") + + +def task_regenerate_permissions(options): + for app in ['ishtar_common', 'archaeological_operations', + 'archaeological_context_records', + 'archaeological_finds', 'archaeological_warehouse']: + create_permissions(apps.get_app_config(app)) + + +def task_missing_parcels(options): + quiet = options.get("quiet", False) Parcel = apps.get_model("archaeological_operations", "Parcel") q = Parcel.objects.filter(context_record__isnull=False, operation=None) nb = q.count() - if not nb: - if not quiet: - sys.stdout.write("No parcel to fix.\n") - return for idx, parcel in enumerate(q.all()): if not quiet: sys.stdout.write(f"\r[{percent(idx, nb)}] {idx + 1}/{nb}") @@ -175,6 +244,13 @@ def task_missing_parcels(quiet=False, log=False): parcel.save() if not quiet: sys.stdout.write("\n") + if not options.get("clean", False): + return + q = Parcel.objects.filter(associated_file=None, operation=None) + nb = q.count() + q.delete() + if not quiet: + sys.stdout.write(f"{nb} orphan parcel deleted.\n") def percent(current, total): @@ -186,21 +262,34 @@ def get_time(): TASKS = { - "main_image": { + "fix_main_image": { "help": "for items with images and no main image, put the first one created as a main image", "action": task_main_image, }, - "cached_label": { - "help": "regenerate cached label on all tables", + "fix_missing_parcels": { + "help": "fix lost parcel association on operation from context records. " + "With --clean option delete orphan parcels.", + "action": task_missing_parcels, + }, + "update_cached_label": { + "help": "regenerate cached label", "action": task_check_cached_label, }, + "update_permissions": { + "help": "regenerate basic model permissions", + "action": task_regenerate_permissions, + }, + "update_qrcodes": { + "help": "regenerate qrcodes", + "action": task_regenerate_qrcodes, + }, "update_search_vector": { - "help": "regenerate search vectors on all tables", + "help": "regenerate search vectors", "action": task_update_search_vectors, }, - "operation_missing_parcels": { - "help": "fix lost parcel association on operation from context records.", - "action": task_missing_parcels, + "update_external_ids": { + "help": "regenerate external and complete ID", + "action": task_update_external_id, }, } @@ -225,23 +314,33 @@ class Command(BaseCommand): [f"{k} - {TASKS[k]['help']}" for k in sorted(TASKS.keys())] ) parser.add_argument("task", help=task_help) + parser.add_argument( + '--model', type=str, default='', dest='model', + help='Specific model to update') parser.add_argument( "--quiet", dest="quiet", action="store_true", help="Quiet output" ) parser.add_argument( "--log", dest="log", action="store_true", help="Log into a file" ) + parser.add_argument( + '--no-https', dest='no-https', action='store_true', + default=False, + help="[update_qrcodes] Use this parameter if https is not available") + parser.add_argument( + '--clean', dest='clean', action='store_true', + default=False, + help="[operation_missing_parcels] Delete orphan parcel after fix") def handle(self, *args, **options): if options["task"] not in TASKS.keys(): msg = f"{options['task']} is not a valid task. Available tasks are:\n" msg += "\n".join(TASKS.keys()) raise CommandError(msg) - log = options["log"] quiet = options["quiet"] if not quiet: sys.stdout.write(f"[{get_time()}] Processing task {options['task']}\n") - errors = TASKS[options["task"]]["action"](quiet=quiet, log=log) + errors = TASKS[options["task"]]["action"](options) if not errors: if not quiet: sys.stdout.write(f"[{get_time()}] Task {options['task']} finished\n") diff --git a/ishtar_common/management/commands/ishtar_migrate_odts.py b/ishtar_common/management/commands/ishtar_migrate_odts.py deleted file mode 100644 index fe7836f0d..000000000 --- a/ishtar_common/management/commands/ishtar_migrate_odts.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (C) 2017 Étienne Loks - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -# See the file COPYING for details. - -from django.core.management.base import BaseCommand -from optparse import make_option -import sys - -from ishtar_common.models import DocumentTemplate -from ishtar_common.utils import BColors - - -class Command(BaseCommand): - help = "Update ODT templates from v1 to v2" - option_list = BaseCommand.option_list + ( - make_option('--quiet', - action='store_true', - dest='quiet', - default=False, - help='Proceed silently with no interactive input.'), - ) - - def interactive_start(self): - sys.stdout.write( - BColors.HEADER + BColors.BOLD + - "Update ODT templates from v1 to v2\n") - sys.stdout.write( - BColors.ENDC + BColors.WARNING + - "This script need to be run only once. Running it on already " - "migrated ODT files may be a source of error.\n") - sys.stdout.write(BColors.ENDC) - yes = None - while yes != "yes": - sys.stdout.write( - "Are you sure you want to proceed? (yes/[n])\n") - yes = input() - if not yes or yes == "n": - sys.stdout.write(BColors.FAIL + "Aborting\n") - sys.stdout.write(BColors.ENDC) - sys.exit() - - def handle(self, *args, **options): - quiet = options['quiet'] - if not quiet: - self.interactive_start() - q = DocumentTemplate.objects - nb = q.count() - len_of_nb = str(len(str(nb))) - if not quiet: - sys.stdout.write(BColors.OKGREEN) - - errors = [] - for idx, document in enumerate(q.all()): - if not quiet: - sys.stdout.write( - ("Processing {:" + len_of_nb + "d}/{}\r").format( - idx + 1, nb)) - sys.stdout.flush() - try: - document.convert_from_v1() - except IOError as e: - errors.append("Document ({}): ".format(document.pk) + - str(e)) - if errors: - sys.stdout.write(BColors.FAIL + "Error while processing:\n") - for error in errors: - sys.stdout.write("* {}\n".format(error)) - - sys.stdout.write(BColors.ENDC) - print("\n\n") - diff --git a/ishtar_common/management/commands/ishtar_migrate_srid.py b/ishtar_common/management/commands/ishtar_migrate_srid.py deleted file mode 100644 index 19de96cb8..000000000 --- a/ishtar_common/management/commands/ishtar_migrate_srid.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (C) 2022 Étienne Loks - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -# See the file COPYING for details. - -from django.core.management.base import BaseCommand -from optparse import make_option -import sys - -from ishtar_common.models import GeoVectorData, IshtarSiteProfile -from ishtar_common.utils import BColors - - -class Command(BaseCommand): - help = "Update SRID - change from profile" - option_list = BaseCommand.option_list + ( - make_option('--quiet', - action='store_true', - dest='quiet', - default=False, - help='Proceed silently with no interactive input.'), - ) - - def interactive_start(self): - sys.stdout.write( - BColors.HEADER + BColors.BOLD + - "Update to new SRID\n") - sys.stdout.write(BColors.ENDC) - yes = None - while yes != "yes": - sys.stdout.write( - "Are you sure you want to proceed? (yes/[n])\n") - yes = input() - if not yes or yes == "n": - sys.stdout.write(BColors.FAIL + "Aborting\n") - sys.stdout.write(BColors.ENDC) - sys.exit() - - def handle(self, *args, **options): - quiet = options['quiet'] - if not quiet: - self.interactive_start() - profile = IshtarSiteProfile.get_current_profile() - if not profile.srs or not profile.srs.srid: - sys.stdout.write(BColors.FAIL + "No SRS set\n") - sys.stdout.write(BColors.ENDC) - GeoVectorData.migrate_srid(profile.srs.srid) - diff --git a/ishtar_common/management/commands/load_towns.py b/ishtar_common/management/commands/load_towns.py deleted file mode 100644 index 00a1b8477..000000000 --- a/ishtar_common/management/commands/load_towns.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import csv -import datetime, time - -from django.conf import settings -from django.core.management.base import BaseCommand, CommandError - -from ishtar_common import tasks - -class Command(BaseCommand): - help = "Load french towns" - - def handle(self, *args, **options): - self.stdout.write("* Loading towns\n") - self.stdout.flush() - created, updated = tasks.load_towns() - self.stdout.write("%d towns created, %s towns updated\n\n" % (created, - updated)) - self.stdout.flush() - diff --git a/ishtar_common/management/commands/migrate_relations_cache_tables.py b/ishtar_common/management/commands/migrate_relations_cache_tables.py new file mode 100644 index 000000000..3e2dfaef5 --- /dev/null +++ b/ishtar_common/management/commands/migrate_relations_cache_tables.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (C) 2021 Étienne Loks + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# See the file COPYING for details. + +import sys + +from django.core.management.base import BaseCommand + +from django.apps import apps + + +CACHE_TABLES = [ + "archaeological_context_records.ContextRecordTree" +] + + +class Command(BaseCommand): + args = '' + help = 'Update all relations for cache tables' + + def add_arguments(self, parser): + parser.add_argument('table', nargs='?', default=None, + choices=CACHE_TABLES) + parser.add_argument( + '--quiet', dest='quiet', action='store_true', + help='Quiet output') + + def handle(self, *args, **options): + quiet = options['quiet'] + tables = CACHE_TABLES + if options.get("table", None): + table = options.get("table", None) + if table not in CACHE_TABLES: + sys.stdout.write("{} not a valid cache table\n".format(table)) + return + tables = [table] + for table in tables: + if not quiet: + print("* table: {}".format(table)) + app, tablename = table.split(".") + model = apps.get_app_config(app).get_model(tablename) + model.regenerate_all(quiet=quiet) + if not quiet: + sys.stdout.write("\n") diff --git a/ishtar_common/management/commands/migrate_srid.py b/ishtar_common/management/commands/migrate_srid.py new file mode 100644 index 000000000..19de96cb8 --- /dev/null +++ b/ishtar_common/management/commands/migrate_srid.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (C) 2022 Étienne Loks + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# See the file COPYING for details. + +from django.core.management.base import BaseCommand +from optparse import make_option +import sys + +from ishtar_common.models import GeoVectorData, IshtarSiteProfile +from ishtar_common.utils import BColors + + +class Command(BaseCommand): + help = "Update SRID - change from profile" + option_list = BaseCommand.option_list + ( + make_option('--quiet', + action='store_true', + dest='quiet', + default=False, + help='Proceed silently with no interactive input.'), + ) + + def interactive_start(self): + sys.stdout.write( + BColors.HEADER + BColors.BOLD + + "Update to new SRID\n") + sys.stdout.write(BColors.ENDC) + yes = None + while yes != "yes": + sys.stdout.write( + "Are you sure you want to proceed? (yes/[n])\n") + yes = input() + if not yes or yes == "n": + sys.stdout.write(BColors.FAIL + "Aborting\n") + sys.stdout.write(BColors.ENDC) + sys.exit() + + def handle(self, *args, **options): + quiet = options['quiet'] + if not quiet: + self.interactive_start() + profile = IshtarSiteProfile.get_current_profile() + if not profile.srs or not profile.srs.srid: + sys.stdout.write(BColors.FAIL + "No SRS set\n") + sys.stdout.write(BColors.ENDC) + GeoVectorData.migrate_srid(profile.srs.srid) + diff --git a/ishtar_common/management/commands/process_generate_merge_candidates.py b/ishtar_common/management/commands/process_generate_merge_candidates.py new file mode 100644 index 000000000..106ce8e13 --- /dev/null +++ b/ishtar_common/management/commands/process_generate_merge_candidates.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (C) 2014 Étienne Loks + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# See the file COPYING for details. + +import sys + +from django.core.management.base import BaseCommand, CommandError +from django.core.exceptions import ObjectDoesNotExist + +import ishtar_common.models as models + + +class Command(BaseCommand): + args = '' + help = 'Regenerate merge candidates' + + def handle(self, *args, **options): + for model in [models.Person, models.Organization]: + sys.stdout.write('\n* %s treatment\n' % str(model)) + q = model.objects + total = q.count() + for idx, item in enumerate(q.all()): + sys.stdout.write('\r\t %d/%d' % (idx, total)) + sys.stdout.flush() + item.generate_merge_candidate() + sys.stdout.write('\nSuccessfully generation of merge candidates\n') diff --git a/ishtar_common/management/commands/process_reassociate_similar_images.py b/ishtar_common/management/commands/process_reassociate_similar_images.py new file mode 100644 index 000000000..3070650b8 --- /dev/null +++ b/ishtar_common/management/commands/process_reassociate_similar_images.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import csv +import datetime +import hashlib +import sys + +from django.core.management.base import BaseCommand + +from ishtar_common.models import Document + +BLOCKSIZE = 65536 + + +def get_hexdigest(filename): + m = hashlib.sha256() + with open(filename, 'rb') as afile: + buf = afile.read(BLOCKSIZE) + while len(buf) > 0: + m.update(buf) + buf = afile.read(BLOCKSIZE) + return m.hexdigest() + + +class Command(BaseCommand): + help = 'Re-associate similar images in the database' + + def add_arguments(self, parser): + parser.add_argument( + '--merge-title', type=str, default='', dest='merged-title', + help='If specified when title differs the given title will be ' + 'used.') + parser.add_argument( + '--output-path', type=str, default='', dest='output-path', + help='Output path for results CSV files. Default to current path.') + parser.add_argument( + '--ignore-reference', dest='ignore-reference', action='store_true', + help='Ignore the reference on diff between models.') + parser.add_argument( + '--delete-missing', dest='delete-missing', action='store_true', + default=False, help='Delete document with missing images.') + parser.add_argument( + '--quiet', dest='quiet', action='store_true', + help='Quiet output.') + + def handle(self, *args, **options): + quiet = options['quiet'] + ignore_ref = options['ignore-reference'] + delete_missing = options['delete-missing'] + merged_title = options['merged-title'] + output_path = options['output-path'] + + q = Document.objects.filter(image__isnull=False).exclude( + image='') + hashes = {} + missing_images = [] + count = q.count() + out = sys.stdout + if not quiet: + out.write("* {} images\n".format(count)) + for idx, doc in enumerate(q.all()): + if not quiet: + out.write("\r* hashes calculation: {} %".format( + int(float(idx + 1) / count * 100))) + out.flush() + path = doc.image.path + try: + hexdigest = get_hexdigest(path) + except IOError: + missing_images.append(doc.pk) + continue + if hexdigest not in hashes: + hashes[hexdigest] = [] + hashes[hexdigest].append(doc.pk) + nb_missing_images = len(missing_images) + if not quiet: + out.write("\n* {} missing images\n".format(nb_missing_images)) + + if missing_images and delete_missing: + for nb, idx in enumerate(missing_images): + if not quiet: + out.write( + "\r* delete document with missing images: {} %".format( + int(float(nb + 1) / nb_missing_images * 100))) + out.flush() + doc = Document.objects.get(pk=idx) + doc.delete() + if not quiet: + out.write("\n") + + attributes = [ + 'title', 'associated_file', 'internal_reference', 'source_type', + 'support_type', 'format_type', 'scale', + 'authors_raw', 'associated_url', 'receipt_date', 'creation_date', + 'receipt_date_in_documentation', 'item_number', 'description', + 'comment', 'additional_information', 'duplicate' + ] + if not ignore_ref: + attributes.append('reference') + + m2ms = ['authors', 'licenses'] + + nb_conflicted_items = 0 + nb_merged_items = 0 + distinct_image = 0 + conflicts = [] + merged = [] + + count = len(hashes) + + for idx, hash in enumerate(hashes): + if not quiet: + out.write("\r* merge similar images: {} %".format( + int(float(idx + 1) / count * 100))) + out.flush() + if len(hashes[hash]) < 2: + distinct_image += 1 + continue + items = [Document.objects.get(pk=pk) for pk in hashes[hash]] + ref_item = items[0] + other_items = items[1:] + + for item in other_items: + ref_item = Document.objects.get(pk=ref_item.pk) + conflicted_values = [] + for attr in attributes: + ref_value = getattr(ref_item, attr) + other_value = getattr(item, attr) + if not other_value: + continue + if ref_value: + if other_value != ref_value: + if attr == 'title' and merged_title: + setattr(ref_item, 'title', merged_title) + else: + conflicted_values.append( + (attr, ref_value, other_value) + ) + else: + setattr(ref_item, attr, other_value) + + base_csv = [ + ref_item.pk, + ref_item.reference.encode('utf-8') if + ref_item.reference else "", + ref_item.cache_related_label.encode('utf-8') if + ref_item.cache_related_label else "", + ref_item.image.name.encode('utf-8'), + item.pk, + item.reference.encode('utf-8') if + item.reference else "", + item.cache_related_label.encode('utf-8') if + item.cache_related_label else "", + item.image.name.encode('utf-8'), + ] + if conflicted_values: + nb_conflicted_items += 1 + for attr, ref_value, other_value in conflicted_values: + conflicts.append(base_csv + [ + attr, str(ref_value).encode('utf-8'), + str(other_value).encode('utf-8') + ]) + continue + + merged.append(base_csv) + + for m2m in m2ms: + for m2 in getattr(item, m2m).all(): + if m2 not in getattr(ref_item, m2m).all(): + getattr(ref_item, m2m).add(m2) + + for rel_attr in Document.RELATED_MODELS: + ref_rel_items = [ + r.pk for r in getattr(ref_item, rel_attr).all()] + for rel_item in getattr(item, rel_attr).all(): + if rel_item.pk not in ref_rel_items: + getattr(ref_item, rel_attr).add(rel_item) + + ref_item.skip_history_when_saving = True + ref_item.save() + item.delete() + nb_merged_items += 1 + if not quiet: + out.write("\n") + + n = datetime.datetime.now().isoformat().split('.')[0].replace(':', '-') + if conflicts: + filename = output_path + "{}-conflict.csv".format(n) + with open(filename, 'w') as csvfile: + writer = csv.writer(csvfile) + writer.writerow( + ["Document 1 - pk", "Document 1 - Ref", + "Document 1 - related", "Document 1 - image path", + "Document 2 - pk", "Document 2 - Ref", + "Document 2 - related", "Document 2 - image path", + "Attribute", "Document 1 - value", "Document 2 - value" + ] + ) + for conflict in conflicts: + writer.writerow(conflict) + if not quiet: + out.write("* {} conflicted items ({})\n".format( + nb_conflicted_items, filename)) + if merged: + filename = output_path + "{}-merged.csv".format(n) + with open(filename, 'w') as csvfile: + writer = csv.writer(csvfile) + writer.writerow( + ["Document 1 - pk", "Document 1 - Ref", + "Document 1 - related", "Document 1 - image path", + "Document 2 - pk", "Document 2 - Ref", + "Document 2 - related", "Document 2 - image path", + ] + ) + for merge in merged: + writer.writerow(merge) + if not quiet: + out.write("* {} merged items ({})\n".format(nb_merged_items, + filename)) + if not quiet: + out.write("* {} distinct images\n".format(distinct_image)) diff --git a/ishtar_common/management/commands/reassociate_similar_images.py b/ishtar_common/management/commands/reassociate_similar_images.py deleted file mode 100644 index 9e52bc6f6..000000000 --- a/ishtar_common/management/commands/reassociate_similar_images.py +++ /dev/null @@ -1,232 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import csv -import datetime -import hashlib -import sys - -from django.core.management.base import BaseCommand - -from ishtar_common.models import Document - -BLOCKSIZE = 65536 - - -def get_hexdigest(filename): - m = hashlib.sha256() - with open(filename, 'rb') as afile: - buf = afile.read(BLOCKSIZE) - while len(buf) > 0: - m.update(buf) - buf = afile.read(BLOCKSIZE) - return m.hexdigest() - - -class Command(BaseCommand): - help = 'Re-associate similar images in the database' - - def add_arguments(self, parser): - parser.add_argument( - '--merge-title', type=str, default='', dest='merged-title', - help='If specified when title differs the given title will be ' - 'used.') - parser.add_argument( - '--output-path', type=str, default='', dest='output-path', - help='Output path for results CSV files. Default to current path.') - parser.add_argument( - '--ignore-reference', dest='ignore-reference', action='store_true', - help='Ignore the reference on diff between models.') - parser.add_argument( - '--delete-missing', dest='delete-missing', action='store_true', - default=False, help='Delete document with missing images.') - parser.add_argument( - '--quiet', dest='quiet', action='store_true', - help='Quiet output.') - - def handle(self, *args, **options): - quiet = options['quiet'] - ignore_ref = options['ignore-reference'] - delete_missing = options['delete-missing'] - merged_title = options['merged-title'] - output_path = options['output-path'] - - q = Document.objects.filter(image__isnull=False).exclude( - image='') - hashes = {} - missing_images = [] - count = q.count() - out = sys.stdout - if not quiet: - out.write("* {} images\n".format(count)) - for idx, doc in enumerate(q.all()): - if not quiet: - out.write("\r* hashes calculation: {} %".format( - int(float(idx + 1) / count * 100))) - out.flush() - path = doc.image.path - try: - hexdigest = get_hexdigest(path) - except IOError: - missing_images.append(doc.pk) - continue - if hexdigest not in hashes: - hashes[hexdigest] = [] - hashes[hexdigest].append(doc.pk) - nb_missing_images = len(missing_images) - if not quiet: - out.write("\n* {} missing images\n".format(nb_missing_images)) - - if missing_images and delete_missing: - for nb, idx in enumerate(missing_images): - if not quiet: - out.write( - "\r* delete document with missing images: {} %".format( - int(float(nb + 1) / nb_missing_images * 100))) - out.flush() - doc = Document.objects.get(pk=idx) - doc.delete() - if not quiet: - out.write("\n") - - attributes = [ - 'title', 'associated_file', 'internal_reference', 'source_type', - 'support_type', 'format_type', 'scale', - 'authors_raw', 'associated_url', 'receipt_date', 'creation_date', - 'receipt_date_in_documentation', 'item_number', 'description', - 'comment', 'additional_information', 'duplicate' - ] - if not ignore_ref: - attributes.append('reference') - - m2ms = ['authors', 'licenses'] - - nb_conflicted_items = 0 - nb_merged_items = 0 - distinct_image = 0 - conflicts = [] - merged = [] - - count = len(hashes) - - for idx, hash in enumerate(hashes): - if not quiet: - out.write("\r* merge similar images: {} %".format( - int(float(idx + 1) / count * 100))) - out.flush() - if len(hashes[hash]) < 2: - distinct_image += 1 - continue - items = [Document.objects.get(pk=pk) for pk in hashes[hash]] - ref_item = items[0] - other_items = items[1:] - - for item in other_items: - ref_item = Document.objects.get(pk=ref_item.pk) - conflicted_values = [] - for attr in attributes: - ref_value = getattr(ref_item, attr) - other_value = getattr(item, attr) - if not other_value: - continue - if ref_value: - if other_value != ref_value: - if attr == 'title' and merged_title: - setattr(ref_item, 'title', merged_title) - else: - conflicted_values.append( - (attr, ref_value, other_value) - ) - else: - setattr(ref_item, attr, other_value) - - base_csv = [ - ref_item.pk, - ref_item.reference.encode('utf-8') if - ref_item.reference else "", - ref_item.cache_related_label.encode('utf-8') if - ref_item.cache_related_label else "", - ref_item.image.name.encode('utf-8'), - item.pk, - item.reference.encode('utf-8') if - item.reference else "", - item.cache_related_label.encode('utf-8') if - item.cache_related_label else "", - item.image.name.encode('utf-8'), - ] - if conflicted_values: - nb_conflicted_items += 1 - for attr, ref_value, other_value in conflicted_values: - conflicts.append(base_csv + [ - attr, str(ref_value).encode('utf-8'), - str(other_value).encode('utf-8') - ]) - continue - - merged.append(base_csv) - - for m2m in m2ms: - for m2 in getattr(item, m2m).all(): - if m2 not in getattr(ref_item, m2m).all(): - getattr(ref_item, m2m).add(m2) - - for rel_attr in Document.RELATED_MODELS: - ref_rel_items = [ - r.pk for r in getattr(ref_item, rel_attr).all()] - for rel_item in getattr(item, rel_attr).all(): - if rel_item.pk not in ref_rel_items: - getattr(ref_item, rel_attr).add(rel_item) - - ref_item.skip_history_when_saving = True - ref_item.save() - item.delete() - nb_merged_items += 1 - if not quiet: - out.write("\n") - - n = datetime.datetime.now().isoformat().split('.')[0].replace(':', '-') - if conflicts: - filename = output_path + "{}-conflict.csv".format(n) - with open(filename, 'w') as csvfile: - writer = csv.writer(csvfile) - writer.writerow( - ["Document 1 - pk", "Document 1 - Ref", - "Document 1 - related", "Document 1 - image path", - "Document 2 - pk", "Document 2 - Ref", - "Document 2 - related", "Document 2 - image path", - "Attribute", "Document 1 - value", "Document 2 - value" - ] - ) - for conflict in conflicts: - writer.writerow(conflict) - if not quiet: - out.write("* {} conflicted items ({})\n".format( - nb_conflicted_items, filename)) - if merged: - filename = output_path + "{}-merged.csv".format(n) - with open(filename, 'w') as csvfile: - writer = csv.writer(csvfile) - writer.writerow( - ["Document 1 - pk", "Document 1 - Ref", - "Document 1 - related", "Document 1 - image path", - "Document 2 - pk", "Document 2 - Ref", - "Document 2 - related", "Document 2 - image path", - ] - ) - for merge in merged: - writer.writerow(merge) - if not quiet: - out.write("* {} merged items ({})\n".format(nb_merged_items, - filename)) - if not quiet: - out.write("* {} distinct images\n".format(distinct_image)) - - - - - - - - - - diff --git a/ishtar_common/management/commands/regenerate_external_id.py b/ishtar_common/management/commands/regenerate_external_id.py deleted file mode 100644 index 9691cba27..000000000 --- a/ishtar_common/management/commands/regenerate_external_id.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (C) 2013-2018 Étienne Loks - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -# See the file COPYING for details. - -import sys - -from django.core.management.base import BaseCommand, CommandError - -from django.apps import apps - - -class Command(BaseCommand): - help = 'Regenerate external id and complete_identifier.' - - def add_arguments(self, parser): - parser.add_argument('model', nargs='+') - - def handle(self, *args, **options): - for app_model_name in options['model']: - try: - capp, model_name = app_model_name.split(".") - except ValueError: - raise CommandError( - "Model argument must be \"app_name.model_name\". Example: " - "archaeological_operations.Operation" - ) - ok = False - for app in ['ishtar_common', 'archaeological_operations', - 'archaeological_context_records', - 'archaeological_finds', 'archaeological_warehouse']: - if capp != app: - continue - for model in apps.get_app_config(app).get_models(): - if model.__name__ != model_name: - continue - if not bool( - [k for k in dir(model) if k == "external_id"]): - continue - msg = "* processing {} - {}:".format( - app, model._meta.verbose_name) - ln = model.objects.count() - for idx, item in enumerate(model.objects.all()): - item.skip_history_when_saving = True - item.external_id = "" - item.complete_identifier = "" - item._no_move = True - cmsg = "\r{} {}/{}".format(msg, idx + 1, ln) - sys.stdout.write(cmsg) - sys.stdout.flush() - item.save() - sys.stdout.write("\n") - ok = True - if not ok: - raise CommandError( - "{} not found or have no external ID".format( - app_model_name)) diff --git a/ishtar_common/management/commands/regenerate_permissions.py b/ishtar_common/management/commands/regenerate_permissions.py deleted file mode 100644 index 9cd8b2277..000000000 --- a/ishtar_common/management/commands/regenerate_permissions.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (C) 2013-2017 Étienne Loks - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -# See the file COPYING for details. - - -from django.core.management.base import BaseCommand - -from django.contrib.auth.management import create_permissions -from django.apps import apps - - -class Command(BaseCommand): - args = '' - help = 'Regenerate permissions' - - def handle(self, *args, **options): - for app in ['ishtar_common', 'archaeological_operations', - 'archaeological_context_records', - 'archaeological_finds', 'archaeological_warehouse']: - create_permissions(apps.get_app_config(app)) diff --git a/ishtar_common/management/commands/regenerate_qrcodes.py b/ishtar_common/management/commands/regenerate_qrcodes.py deleted file mode 100644 index d915d2e21..000000000 --- a/ishtar_common/management/commands/regenerate_qrcodes.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (C) 2019 Étienne Loks - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -# See the file COPYING for details. - -import sys -import tempfile -import shutil - -from django.core.management.base import BaseCommand -from django.core.exceptions import FieldDoesNotExist - -from django.apps import apps - - -APPS = ['ishtar_common', 'archaeological_operations', - 'archaeological_context_records', 'archaeological_finds', - 'archaeological_warehouse'] - - -class Command(BaseCommand): - args = '' - help = 'Regenerate QR codes' - - def add_arguments(self, parser): - parser.add_argument('app_name', nargs='?', default=None, - choices=APPS) - parser.add_argument('model_name', nargs='?', default=None) - parser.add_argument( - '--no-https', dest='no-https', action='store_true', - default=False, - help='Use this parameter if https is not available.') - - def handle(self, *args, **options): - limit = options['app_name'] - model_name = options['model_name'] - secure = not options['no-https'] - if model_name: - model_name = model_name.lower() - for app in APPS: - if limit and app != limit: - continue - print("* app: {}".format(app)) - for model in apps.get_app_config(app).get_models(): - if model_name and model.__name__.lower() != model_name: - continue - if model.__name__.startswith('Historical'): - continue - try: - model._meta.get_field('qrcode') - except FieldDoesNotExist: - continue - msg = "-> processing {}: ".format(model._meta.verbose_name) - ln = model.objects.count() - tmpdir = tempfile.mkdtemp("-qrcode") - for idx, object in enumerate(model.objects.all()): - object.skip_history_when_saving = True - object._no_move = True - cmsg = "\r{} {}/{}".format(msg, idx + 1, ln) - sys.stdout.write(cmsg) - sys.stdout.flush() - object.generate_qrcode(secure=secure, tmpdir=tmpdir) - shutil.rmtree(tmpdir) - sys.stdout.write("\n") diff --git a/ishtar_common/management/commands/regenerate_search_vector_cached_label.py b/ishtar_common/management/commands/regenerate_search_vector_cached_label.py deleted file mode 100644 index 1b9dd24d1..000000000 --- a/ishtar_common/management/commands/regenerate_search_vector_cached_label.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (C) 2013-2018 Étienne Loks - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -# See the file COPYING for details. - -import sys - -from django.core.management.base import BaseCommand - -from django.apps import apps - - -APPS = ['ishtar_common', 'archaeological_operations', - 'archaeological_context_records', 'archaeological_finds', - 'archaeological_warehouse'] - - -class Command(BaseCommand): - args = '' - help = 'Regenerate geo, cached labels and search vectors' - - def add_arguments(self, parser): - parser.add_argument('app_name', nargs='?', default=None, - choices=APPS) - parser.add_argument('model_name', nargs='?', default=None) - parser.add_argument( - '--quiet', dest='quiet', action='store_true', - help='Quiet output') - - def handle(self, *args, **options): - quiet = options['quiet'] - limit = options['app_name'] - model_name = options['model_name'] - if model_name: - model_name = model_name.lower() - for app in APPS: - if limit and app != limit: - continue - if not quiet: - print("* app: {}".format(app)) - for model in apps.get_app_config(app).get_models(): - if model_name and model.__name__.lower() != model_name: - continue - if model.__name__.startswith('Historical'): - continue - if not bool( - [k for k in dir(model) - if k.startswith('_generate_') or - k == "search_vector"]): - continue - msg = "-> processing {}: ".format(model._meta.verbose_name) - ln = model.objects.count() - for idx, obj_id in enumerate(model.objects.values('pk').all()): - obj = model.objects.get(pk=obj_id['pk']) - obj.skip_history_when_saving = True - obj._no_move = True - cmsg = "\r{} {}/{}".format(msg, idx + 1, ln) - if not quiet: - sys.stdout.write(cmsg) - sys.stdout.flush() - obj.save() - if not quiet: - sys.stdout.write("\n") diff --git a/ishtar_common/management/commands/relations_update_cache_tables.py b/ishtar_common/management/commands/relations_update_cache_tables.py deleted file mode 100644 index 3e2dfaef5..000000000 --- a/ishtar_common/management/commands/relations_update_cache_tables.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (C) 2021 Étienne Loks - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -# See the file COPYING for details. - -import sys - -from django.core.management.base import BaseCommand - -from django.apps import apps - - -CACHE_TABLES = [ - "archaeological_context_records.ContextRecordTree" -] - - -class Command(BaseCommand): - args = '' - help = 'Update all relations for cache tables' - - def add_arguments(self, parser): - parser.add_argument('table', nargs='?', default=None, - choices=CACHE_TABLES) - parser.add_argument( - '--quiet', dest='quiet', action='store_true', - help='Quiet output') - - def handle(self, *args, **options): - quiet = options['quiet'] - tables = CACHE_TABLES - if options.get("table", None): - table = options.get("table", None) - if table not in CACHE_TABLES: - sys.stdout.write("{} not a valid cache table\n".format(table)) - return - tables = [table] - for table in tables: - if not quiet: - print("* table: {}".format(table)) - app, tablename = table.split(".") - model = apps.get_app_config(app).get_model(tablename) - model.regenerate_all(quiet=quiet) - if not quiet: - sys.stdout.write("\n") diff --git a/ishtar_common/management/commands/update_external_ids.py b/ishtar_common/management/commands/update_external_ids.py deleted file mode 100644 index f69a865d7..000000000 --- a/ishtar_common/management/commands/update_external_ids.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import sys - -from django.core.management.base import BaseCommand - -from archaeological_operations.models import Parcel -from archaeological_context_records.models import ContextRecord -from archaeological_finds.models import BaseFind, Find - - -class Command(BaseCommand): - help = "./manage.py ishtar_execute_admin_tasks\n\n"\ - "Launch pending administration tasks." - - def handle(self, *args, **options): - for model in [Parcel, ContextRecord, BaseFind, Find]: - updated = 0 - print("* {}".format(model)) - total = model.objects.count() - for idx, item in enumerate(model.objects.all()): - sys.stdout.write("\r{}/{} ".format(idx, total)) - sys.stdout.flush() - updated += 1 if item.update_external_id(save=True) else 0 - print("\rupdated: {} ".format(updated)) diff --git a/ishtar_common/management/commands/update_search_vectors.py b/ishtar_common/management/commands/update_search_vectors.py deleted file mode 100644 index a07da721e..000000000 --- a/ishtar_common/management/commands/update_search_vectors.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import sys - -from django.core.management.base import BaseCommand -import django.apps - - -class Command(BaseCommand): - help = "./manage.py update_search_vectors\n\n"\ - "Update full texte search vectors." - - def add_arguments(self, parser): - parser.add_argument( - '--model', type=str, default='', dest='model', - help='Specific model to update') - parser.add_argument( - '--quiet', dest='quiet', action='store_true', - help='Quiet output') - - def handle(self, *args, **options): - quiet = options['quiet'] - nb_total = 0 - for model in django.apps.apps.get_models(): - if options['model'] and model.__name__ != options['model']: - continue - if hasattr(model, "update_search_vector") and \ - (getattr(model, "BASE_SEARCH_VECTORS", None) or - getattr(model, "INT_SEARCH_VECTORS", None) or - getattr(model, "M2M_SEARCH_VECTORS", None) or - getattr(model, "PARENT_SEARCH_VECTORS", None)): - if not quiet: - self.stdout.write("\n* update {}".format(model)) - total = model.objects.count() - idx = 0 - for idx, item in enumerate(model.objects.all()): - if not quiet: - sys.stdout.write("\r{}/{} ".format(idx + 1, total)) - sys.stdout.flush() - try: - item.update_search_vector() - except: - pass - nb_total += idx - if not quiet: - self.stdout.write("\n") - self.stdout.write("{} items updated\n".format(nb_total)) diff --git a/ishtar_common/management/commands/update_specific_importers.py b/ishtar_common/management/commands/update_specific_importers.py deleted file mode 100644 index 9a13e3f3e..000000000 --- a/ishtar_common/management/commands/update_specific_importers.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from optparse import make_option - -from django.core.management.base import BaseCommand - -IMPORTERS = [] - -from archaeological_files.data_importer import FileImporterSraPdL -IMPORTERS.append(FileImporterSraPdL) - - -class Command(BaseCommand): - help = "Update each specific importer" - option_list = list(BaseCommand.option_list) + [ - make_option( - '--force', - action='store_true', - dest='force', - default=False, - help='Force the deletion of existing importers. ' - 'ATTENTION: all associated imports and all associated items ' - 'will be deleted.')] - - def handle(self, *args, **options): - for importer in IMPORTERS: - response = importer()._create_models(force=options['force']) - if response: - self.stdout.write("%s configured\n" % importer.__name__) - self.stdout.flush() diff --git a/ishtar_common/models.py b/ishtar_common/models.py index 80a6a9141..00d94a1f2 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -1066,7 +1066,7 @@ class IshtarSiteProfile(models.Model, Cached): "(for instance: complex statigraphic relations), set it to " '"Cache tables" in order to use static cache tables. Do not ' "forget to update theses table with the " - '"relations_update_cache_tables" manage.py command.' + '"migrate_relations_cache_tables" manage.py command.' ), ) config = models.CharField( @@ -1502,7 +1502,7 @@ class IshtarSiteProfile(models.Model, Cached): null=True, help_text=_( "Set it to the most used spatial reference system. Warning: after change " - "launch the ishtar_migrate_srid script." + "launch the migrate_srid script." ), on_delete=models.SET_NULL, related_name="profile_srs" diff --git a/ishtar_common/tasks.py b/ishtar_common/tasks.py index 03b8a6338..fe6b7045a 100644 --- a/ishtar_common/tasks.py +++ b/ishtar_common/tasks.py @@ -182,39 +182,6 @@ def launch_export(export_task_id): export_task.save() -def load_towns(): - # TODO: remove? - from geodjangofla.models import Commune - - q = None - for dpt_number in settings.ISHTAR_DPTS: - query = Q(insee_com__istartswith=dpt_number) - if q: - q = q | query - else: - q = query - if q: - q = Commune.objects.filter(q) - else: - q = Commune.objects - nb, updated = 0, 0 - for town in q.all(): - surface = town.superficie or 0 - surface = surface * 10000 - defaults = {"name": town.nom_comm, "surface": surface, "center": town.centroid} - town, created = Town.objects.get_or_create( - numero_insee=town.insee_com, defaults=defaults - ) - if created: - nb += 1 - else: - updated += 1 - for k in defaults: - setattr(town, k, defaults[k]) - town.save() - return nb, updated - - def update_towns(): # TODO: remove? nb, updated = 0, 0 diff --git a/locale/fr/LC_MESSAGES/django.po b/locale/fr/LC_MESSAGES/django.po index 18ed8edc9..12a007712 100644 --- a/locale/fr/LC_MESSAGES/django.po +++ b/locale/fr/LC_MESSAGES/django.po @@ -9634,13 +9634,13 @@ msgid "" "If you experience performance problems with complex relations (for instance: " "complex statigraphic relations), set it to \"Cache tables\" in order to use " "static cache tables. Do not forget to update theses table with the " -"\"relations_update_cache_tables\" manage.py command." +"\"migrate_relations_cache_tables\" manage.py command." msgstr "" "Si vous rencontrez des problèmes de performance avec des relations complexes " "(par exemple : des relations statigraphiques complexes), mettez-le sur " "\"Tables de cache\" afin d'utiliser des tables de cache statiques. N'oubliez " "pas de mettre à jour ces tables avec la commande manage.py " -"\"relations_update_cache_tables\"." +"\"migrate_relations_cache_tables\"." #: ishtar_common/models.py:1073 msgid "Alternate configuration" -- cgit v1.2.3