diff options
15 files changed, 615 insertions, 11 deletions
diff --git a/archaeological_finds/migrations/0054_migrate_main_image.py b/archaeological_finds/migrations/0054_migrate_main_image.py index 1a7f942c4..6d11a5a67 100644 --- a/archaeological_finds/migrations/0054_migrate_main_image.py +++ b/archaeological_finds/migrations/0054_migrate_main_image.py @@ -7,9 +7,10 @@ from ishtar_common.utils_migrations import migrate_main_image def migrate_main_image_script(apps, schema): - migrate_main_image(apps, 'archaeological_finds', 'Find') - migrate_main_image(apps, 'archaeological_finds', 'Treatment') - migrate_main_image(apps, 'archaeological_finds', 'TreatmentFile') + migrate_main_image(apps, 'archaeological_finds', 'Find', verbose=True) + migrate_main_image(apps, 'archaeological_finds', 'Treatment', verbose=True) + migrate_main_image(apps, 'archaeological_finds', 'TreatmentFile', + verbose=True) class Migration(migrations.Migration): diff --git a/ishtar_common/management/commands/media_clean_unused.py b/ishtar_common/management/commands/media_clean_unused.py new file mode 100644 index 000000000..33237c1c7 --- /dev/null +++ b/ishtar_common/management/commands/media_clean_unused.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- + +""" +Copyright (c) 2014 Andrey Kolpakov - MIT License (MIT) +https://github.com/akolpakov/django-unused-media +""" + +import os + +import six.moves +from django.conf import settings +from django.core.management.base import BaseCommand + +from ishtar_common.utils import get_unused_media +from ishtar_common.utils import remove_empty_dirs + + +class Command(BaseCommand): + + help = "Clean unused media files which have no reference in models" + + # verbosity + # 0 means minimal output + # 1 means normal output (default). + # 2 means verbose output + + verbosity = 1 + + def add_arguments(self, parser): + parser.add_argument('--noinput', '--no-input', + dest='interactive', + action='store_false', + default=True, + help='Do not ask confirmation') + + parser.add_argument('-e', '--exclude', + dest='exclude', + action='append', + default=[], + help='Exclude files by mask (only * is supported), can use multiple --exclude') + + parser.add_argument('--remove-empty-dirs', + dest='remove_empty_dirs', + action='store_false', + default=False, + help='Remove empty dirs after files cleanup') + + parser.add_argument('-n', '--dry-run', + dest='dry_run', + action='store_true', + default=False, + help='Dry run without any affect on your data') + + def info(self, message): + if self.verbosity >= 0: + self.stdout.write(message) + + def debug(self, message): + if self.verbosity >= 1: + self.stdout.write(message) + + def _show_files_to_delete(self, unused_media): + self.debug('Files to remove:') + + for f in unused_media: + self.debug(f) + + self.info('Total files will be removed: {}'.format(len(unused_media))) + + def handle(self, *args, **options): + + if 'verbosity' in options: + self.verbosity = options['verbosity'] + + unused_media = get_unused_media(options.get('exclude') or []) + + if not unused_media: + self.info('Nothing to delete. Exit') + return + + if options.get('dry_run'): + self._show_files_to_delete(unused_media) + self.info('Dry run. Exit.') + return + + if options.get('interactive'): + self._show_files_to_delete(unused_media) + + # ask user + question = 'Are you sure you want to remove {} unused files? (y/N)'.format(len(unused_media)) + + if six.moves.input(question).upper() != 'Y': + self.info('Interrupted by user. Exit.') + return + + for f in unused_media: + self.debug('Remove %s' % f) + os.remove(os.path.join(settings.MEDIA_ROOT, f)) + + if options.get('remove_empty_dirs'): + remove_empty_dirs() + + self.info('Done. Total files removed: {}'.format(len(unused_media))) diff --git a/ishtar_common/management/commands/media_find_missing_files.py b/ishtar_common/management/commands/media_find_missing_files.py new file mode 100644 index 000000000..226699842 --- /dev/null +++ b/ishtar_common/management/commands/media_find_missing_files.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (C) 2019 Étienne Loks <etienne.loks_AT_peacefrogsDOTnet> + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# See the file COPYING for details. + +import os.path +import sys + +from ishtar_common.utils import get_used_media, try_fix_file + +from django.core.management.base import BaseCommand + + +class Command(BaseCommand): + args = '' + help = 'Find missing files in media.' + + def add_arguments(self, parser): + parser.add_argument( + '--exclude', nargs='?', dest='exclude', default=None, + help="Field to exclude separated with \",\". Example: " + "ishtar_common.Import.imported_file," + "ishtar_common.Import.imported_images" + ) + parser.add_argument( + '--limit', nargs='?', dest='limit', default=None, + help="Field to limit to separated with \",\"." + ) + parser.add_argument( + '--try-fix', dest='try-fix', action='store_true', + default=False, + help='Try to find file with similar names and copy the file.') + parser.add_argument( + '--find-fix', dest='find-fix', action='store_true', + default=False, + help='Try to find file with similar names and print them.') + + def handle(self, *args, **options): + exclude = options['exclude'].split(',') if options['exclude'] else [] + limit = options['limit'].split(',') if options['limit'] else [] + try_fix = options['try-fix'] + find_fix = options['find-fix'] + if try_fix and find_fix: + self.stdout.write("try-fix and find-fix options are not " + "compatible.\n") + return + + missing = [] + for media in get_used_media(exclude=exclude, limit=limit): + if not os.path.isfile(media): + missing.append(media) + + if try_fix or find_fix: + if find_fix: + self.stdout.write("* potential similar file:\n") + else: + self.stdout.write("* fixes files:\n") + for item in missing[:]: + source_file = try_fix_file(item, make_copy=try_fix) + if source_file: + missing.pop(missing.index(item)) + sys.stdout.write( + "{} <- {}\n".format(item.encode('utf-8'), + source_file.encode('utf-8'))) + + if missing: + if find_fix or try_fix: + self.stdout.write("* missing file with no similar file " + "found:\n") + for item in missing: + sys.stdout.write(item.encode('utf-8') + "\n") + else: + self.stdout.write("No missing files.\n") diff --git a/ishtar_common/management/commands/media_simplify_filenames.py b/ishtar_common/management/commands/media_simplify_filenames.py new file mode 100644 index 000000000..2295431cd --- /dev/null +++ b/ishtar_common/management/commands/media_simplify_filenames.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (C) 2019 Étienne Loks <etienne.loks_AT_peacefrogsDOTnet> + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# See the file COPYING for details. + +import os.path +import sys + + +from django.conf import settings +from django.core.management.base import BaseCommand + +from ishtar_common.utils import find_all_symlink, \ + rename_and_simplify_media_name, get_used_media + + +class Command(BaseCommand): + args = '' + help = 'Simplify name of files in media' + + def add_arguments(self, parser): + parser.add_argument( + '--exclude', nargs='?', dest='exclude', default=None, + help="Field to exclude separated with \",\". Example: " + "ishtar_common.Import.imported_file," + "ishtar_common.Import.imported_images" + ) + parser.add_argument( + '--limit', nargs='?', dest='limit', default=None, + help="Field to limit to separated with \",\"." + ) + + def handle(self, *args, **options): + exclude = options['exclude'].split(',') if options['exclude'] else [] + limit = options['limit'].split(',') if options['limit'] else [] + verbose = options['verbosity'] + + current_links = dict(list(find_all_symlink(settings.MEDIA_ROOT))) + + new_names = {} + if verbose == 2: + sys.stdout.write("* list current media...\n") + sys.stdout.flush() + + media = get_used_media(exclude=exclude, limit=limit, + return_object_and_field=True) + ln = len(media) + for idx, result in enumerate(sorted(media, key=lambda x: x[2])): + if verbose == 2: + nb_point = 10 + sys.stdout.write("* processing {}/{} {}{}\r".format( + idx + 1, ln, (idx % nb_point) * ".", " " * nb_point) + ) + sys.stdout.flush() + obj, field_name, media = result + if not os.path.isfile(media): + # missing file... + continue + if media not in new_names: + new_name, modified = rename_and_simplify_media_name(media) + if not modified: + continue + new_names[media] = new_name + setattr(obj, field_name, new_names[media]) + obj._no_move = True + obj._cached_label_checked = True + obj.skip_history_when_saving = True + obj.save() + if verbose > 2: + sys.stdout.write("{} renamed {}\n".format( + media, new_names[media].split(os.sep)[-1])) + if media in current_links.keys(): + os.remove(current_links[media]) + os.symlink(new_names[media], current_links[media]) + if verbose > 2: + sys.stdout.write( + "symlink {} changed to point to {}\n".format( + current_links[media], new_names[media])) + current_links.pop(media) + if verbose == 2: + sys.stdout.write("\n") diff --git a/ishtar_common/migrations/0081_recreate_m2m_history.py b/ishtar_common/migrations/0081_recreate_m2m_history.py index e26a3f185..8f2779aed 100644 --- a/ishtar_common/migrations/0081_recreate_m2m_history.py +++ b/ishtar_common/migrations/0081_recreate_m2m_history.py @@ -2,6 +2,8 @@ # Generated by Django 1.11.10 on 2019-01-16 11:16 from __future__ import unicode_literals +import sys + from django.db import migrations from ishtar_common.utils_migrations import m2m_historization_init @@ -14,9 +16,17 @@ def recreate_m2m_migrations(apps, schema_editor): history_models = [ ContextRecord, File, Find, Treatment, Operation, ArchaeologicalSite ] + sys.stdout.write("\n") for model in history_models: - for item in model.objects.all(): + q = model.objects + ln = q.count() + for idx, item in enumerate(model.objects.all()): + sys.stdout.write("{}: {}/{}\r".format(model, idx + 1, ln)) + sys.stdout.flush() m2m_historization_init(item) + if ln: + sys.stdout.write("\n") + sys.stdout.write("\n") class Migration(migrations.Migration): diff --git a/ishtar_common/migrations/0083_document_index_external_id.py b/ishtar_common/migrations/0083_document_index_external_id.py index 593962885..4f1dafcf9 100644 --- a/ishtar_common/migrations/0083_document_index_external_id.py +++ b/ishtar_common/migrations/0083_document_index_external_id.py @@ -1,13 +1,19 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2019-01-18 17:51 from __future__ import unicode_literals +import sys from django.db import migrations def gen_index(apps, schema_editor): from ishtar_common.models import Document - for doc in Document.objects.all(): + sys.stdout.write("\n") + q = Document.objects + ln = q.count() + for idx, doc in enumerate(Document.objects.all()): + sys.stdout.write(" * {}/{}\r".format(idx, ln)) + sys.stdout.flush() doc._no_move = True doc.save() diff --git a/ishtar_common/models.py b/ishtar_common/models.py index 425ee2a8d..5db454b14 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -4127,9 +4127,10 @@ class Document(BaseHistorizedItem, OwnPerms, ImageModel): self._get_thumb_name(reference_path)) self.image.name = reference_path[ len(settings.MEDIA_ROOT):] + self._no_move = True self.save(no_path_change=True) except OSError: - # file probably not on harddrive - will be cleaned + # file probably not on HDD - will be cleaned pass continue # create a link diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py index cf9f599c4..f51f9736a 100644 --- a/ishtar_common/tests.py +++ b/ishtar_common/tests.py @@ -45,7 +45,8 @@ from django.test.runner import DiscoverRunner from ishtar_common import models from ishtar_common import views from ishtar_common.apps import admin_site -from ishtar_common.utils import post_save_point, update_data, move_dict_data +from ishtar_common.utils import post_save_point, update_data, move_dict_data, \ + rename_and_simplify_media_name, try_fix_file COMMON_FIXTURES = [ @@ -1657,3 +1658,41 @@ class DashboardTest(TestCase): self.assertEqual( response.status_code, 200, "Reaching dashboard for item: {} return an error.".format(url)) + + +class CleanMedia(TestCase): + + def test_rename(self): + test_names = [ + (u"éofficier2-12-02-04.93_gvK3hAr-1_2m7zZPn-1_nKhh2S2-1_"\ + u"ONmUhfD-1_ymA3gGJ-1_XzJyRx3-1_PhvRcO8-1-thumb_ZwWMKBd.jpg", + u"éofficier2-12-02-04.93-thumb.jpg"), + (u"a_ZwWMKBd.jpg", False), # no rename because too short + (u"hoplala_gvK3hAr_2m7zZPn_nKhh2S2_ZwWMKBd.jpg", + u"hoplala_gvK3hAr_2m7zZPn_nKhh2S2.jpg",), # stop before because + # another file exists + ] + base_dir = os.sep.join([settings.ROOT_PATH, u"..", u"ishtar_common", + u"tests", u"rename"]) + for name, expected in test_names: + name = os.sep.join([base_dir, name]) + new_name, modif = rename_and_simplify_media_name(name, rename=False) + if expected: + self.assertTrue(modif) + self.assertEqual(new_name, os.sep.join([base_dir, expected])) + else: + self.assertFalse(modif) + + def test_try_fix(self): + test_names = [ + (u"hoplala_gvK3hAr_2m7zZPn_nKhh2S2_ZwWMKBd_ZwWMKBd.jpg", + # non existing file + u"hoplala_gvK3hAr_2m7zZPn.jpg",), + ] + base_dir = os.sep.join([settings.ROOT_PATH, u"..", u"ishtar_common", + u"tests", u"rename"]) + for name, expected in test_names: + name = os.sep.join([base_dir, name]) + + found = try_fix_file(name, make_copy=False) + self.assertEqual(found, expected) diff --git a/example_project/media/imported/imported_files_here b/ishtar_common/tests/rename/a_ZwWMKBd.jpg index e69de29bb..e69de29bb 100644 --- a/example_project/media/imported/imported_files_here +++ b/ishtar_common/tests/rename/a_ZwWMKBd.jpg diff --git a/ishtar_common/tests/rename/hoplala_gvK3hAr_2m7zZPn.jpg b/ishtar_common/tests/rename/hoplala_gvK3hAr_2m7zZPn.jpg new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/ishtar_common/tests/rename/hoplala_gvK3hAr_2m7zZPn.jpg diff --git a/ishtar_common/tests/rename/hoplala_gvK3hAr_2m7zZPn_nKhh2S2_ZwWMKBd.jpg b/ishtar_common/tests/rename/hoplala_gvK3hAr_2m7zZPn_nKhh2S2_ZwWMKBd.jpg new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/ishtar_common/tests/rename/hoplala_gvK3hAr_2m7zZPn_nKhh2S2_ZwWMKBd.jpg diff --git a/ishtar_common/tests/rename/éofficier2-12-02-04.93_gvK3hAr-1_2m7zZPn-1_nKhh2S2-1_ONmUhfD-1-thumb.jpg b/ishtar_common/tests/rename/éofficier2-12-02-04.93_gvK3hAr-1_2m7zZPn-1_nKhh2S2-1_ONmUhfD-1-thumb.jpg new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/ishtar_common/tests/rename/éofficier2-12-02-04.93_gvK3hAr-1_2m7zZPn-1_nKhh2S2-1_ONmUhfD-1-thumb.jpg diff --git a/ishtar_common/tests/rename/éofficier2-12-02-04.93_gvK3hAr-1_2m7zZPn-1_nKhh2S2-1_ONmUhfD-1_ymA3gGJ-1_XzJyRx3-1_PhvRcO8-1-thumb_ZwWMKBd.jpg b/ishtar_common/tests/rename/éofficier2-12-02-04.93_gvK3hAr-1_2m7zZPn-1_nKhh2S2-1_ONmUhfD-1_ymA3gGJ-1_XzJyRx3-1_PhvRcO8-1-thumb_ZwWMKBd.jpg new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/ishtar_common/tests/rename/éofficier2-12-02-04.93_gvK3hAr-1_2m7zZPn-1_nKhh2S2-1_ONmUhfD-1_ymA3gGJ-1_XzJyRx3-1_PhvRcO8-1-thumb_ZwWMKBd.jpg diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py index f2fe34631..2156d4f95 100644 --- a/ishtar_common/utils.py +++ b/ishtar_common/utils.py @@ -24,12 +24,16 @@ from itertools import chain import hashlib import os import random +import re import requests import shutil +import six import subprocess +import sys import tempfile from django import forms +from django.apps import apps from django.conf import settings from django.conf.urls import url from django.contrib.contenttypes.models import ContentType @@ -37,7 +41,9 @@ from django.contrib.gis.geos import GEOSGeometry from django.contrib.sessions.backends.db import SessionStore from django.core.cache import cache from django.core.files import File +from django.core.validators import EMPTY_VALUES from django.core.urlresolvers import reverse +from django.db import models from django.http import HttpResponseRedirect from django.utils.datastructures import MultiValueDict as BaseMultiValueDict from django.utils.safestring import mark_safe @@ -966,3 +972,248 @@ def max_size_help(): settings.MAX_UPLOAD_SIZE ) return msg + + +def find_all_symlink(dirname): + for name in os.listdir(dirname): + if name not in (os.curdir, os.pardir): + full = os.path.join(dirname, name) + if os.path.islink(full): + yield full, os.readlink(full) + + +MEDIA_RE = [ + re.compile(r"_[a-zA-Z0-9]{7}\-[0-9]"), + re.compile(r"_[a-zA-Z0-9]{7}"), +] + + +def simplify_name(full_path_name, check_existing=False): + """ + Simplify a file name by removing auto save suffixes + + :param full_path_name: full path name + :param check_existing: prevent to give name of an existing file + :return: + """ + name_exp = full_path_name.split(os.sep) + path = os.sep.join(name_exp[0:-1]) + name = name_exp[-1] + current_name = name[:] + ext = "" + if u"." in name: # remove extension if have one + names = name.split(u".") + name = u".".join(names[0:-1]) + ext = u"." + names[-1] + + while u"_" in name and len(name) > 15: + oldname = name[:] + for regex in MEDIA_RE: + match = None + for m in regex.finditer(name): # get the last match + match = m + if match: + new_name = name.replace(match.group(), '') + full_new_name = os.sep.join([path, new_name + ext]) + if not check_existing or not os.path.isfile(full_new_name): + # do not take the place of another file + name = new_name[:] + break + if oldname == name: + break + return path, current_name, name + ext + + +def rename_and_simplify_media_name(full_path_name, rename=True): + """ + Simplify the name if possible + :param full_path_name: full path name + :param rename: rename file if True (default: True) + :return: new full path name (or old if not changed), modified + """ + if not os.path.exists(full_path_name) or not os.path.isfile(full_path_name): + return full_path_name, False + + path, current_name, name = simplify_name(full_path_name, + check_existing=True) + if current_name == name: + return full_path_name, False + + full_new_name = os.sep.join([path, name]) + if rename: + os.rename(full_path_name, full_new_name) + return full_new_name, True + + +def get_file_fields(): + """ + Get all fields which are inherited from FileField + """ + all_models = apps.get_models() + fields = [] + for model in all_models: + for field in model._meta.get_fields(): + if isinstance(field, models.FileField): + fields.append(field) + return fields + + +def get_used_media(exclude=None, limit=None, + return_object_and_field=False, debug=False): + """ + Get media which are still used in models + + :param exclude: exclude fields, ex: ['ishtar_common.Import.imported_file', + 'ishtar_common.Import.imported_images'] + :param limit: limit to some fields + :param return_object_and_field: return associated object and field name + :return: list of media filename or if return_object_and_field is set to + True return (object, file field name, media filename) + """ + + if return_object_and_field: + media = [] + else: + media = set() + for field in get_file_fields(): + if exclude and unicode(field) in exclude: + continue + if limit and unicode(field) not in limit: + continue + is_null = {'%s__isnull' % field.name: True} + is_empty = {'%s' % field.name: ''} + + storage = field.storage + if debug: + print("") + q = field.model.objects.values('id', field.name)\ + .exclude(**is_empty).exclude(**is_null) + ln = q.count() + for idx, res in enumerate(q): + value = res[field.name] + if debug: + sys.stdout.write("* get_used_media {}: {}/{}\r".format( + field, idx, ln)) + sys.stdout.flush() + if value not in EMPTY_VALUES: + if return_object_and_field: + media.append(( + field.model.objects.get(pk=res['id']), + field.name, + storage.path(value) + )) + else: + media.add(storage.path(value)) + return media + + +def get_all_media(exclude=None, debug=False): + """ + Get all media from MEDIA_ROOT + """ + + if not exclude: + exclude = [] + + media = set() + full_dirs = list(os.walk(six.text_type(settings.MEDIA_ROOT))) + ln_full = len(full_dirs) + for idx_main, full_dir in enumerate(full_dirs): + root, dirs, files = full_dir + ln = len(files) + if debug: + print("") + for idx, name in enumerate(files): + if debug: + sys.stdout.write("* get_all_media {} ({}/{}): {}/{}\r".format( + root.encode('utf-8'), idx_main, ln_full, idx, ln)) + sys.stdout.flush() + path = os.path.abspath(os.path.join(root, name)) + relpath = os.path.relpath(path, settings.MEDIA_ROOT) + in_exclude = False + for e in exclude: + if re.match(r'^%s$' % re.escape(e).replace('\\*', '.*'), + relpath): + in_exclude = True + break + + if not in_exclude: + media.add(path) + else: + if debug: + sys.stdout.write("* get_all_media {} ({}/{})\r".format( + root.encode('utf-8'), idx_main, ln_full)) + return media + + +def get_unused_media(exclude=None): + """ + Get media which are not used in models + """ + + if not exclude: + exclude = [] + + all_media = get_all_media(exclude) + used_media = get_used_media() + + return [x for x in all_media if x not in used_media] + + +def remove_unused_media(): + """ + Remove unused media + """ + remove_media(get_unused_media()) + + +def remove_media(files): + """ + Delete file from media dir + """ + for filename in files: + os.remove(os.path.join(settings.MEDIA_ROOT, filename)) + + +def remove_empty_dirs(path=None): + """ + Recursively delete empty directories; return True if everything was deleted. + """ + + if not path: + path = settings.MEDIA_ROOT + + if not os.path.isdir(path): + return False + + listdir = [os.path.join(path, filename) for filename in os.listdir(path)] + + if all(list(map(remove_empty_dirs, listdir))): + os.rmdir(path) + return True + else: + return False + + +def try_fix_file(filename, make_copy=True): + """ + Try to find a file with a similar name on the same dir. + + :param filename: filename (full path) + :param make_copy: make the copy of the similar file found + :return: name of the similar file found or None + """ + path, current_name, simplified_ref_name = simplify_name( + filename, check_existing=False) + + # check existing files in the path + for file in sorted(list(os.listdir(path))): + full_file = os.sep.join([path, file]) + if not os.path.isfile(full_file): # must be a file + continue + _, _, name = simplify_name(full_file, check_existing=False) + if simplified_ref_name.lower() == name.lower(): + # a candidate is found + if make_copy: + shutil.copy2(full_file, filename) + return file diff --git a/ishtar_common/utils_migrations.py b/ishtar_common/utils_migrations.py index 431fe5bd7..f51465262 100644 --- a/ishtar_common/utils_migrations.py +++ b/ishtar_common/utils_migrations.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import datetime import json import os +import sys from django.core.files import File from django.db import connection @@ -102,11 +103,18 @@ def reinit_last_modified(apps, app_name, models): item.save() -def migrate_main_image(apps, app_name, model_name): +def migrate_main_image(apps, app_name, model_name, verbose=False): model = apps.get_model(app_name, model_name) - for item in model.objects.filter( - documents__image__isnull=False).exclude( - main_image__isnull=False).all(): + q = model.objects.filter(documents__image__isnull=False).exclude( + main_image__isnull=False) + ln = q.count() + if verbose: + sys.stdout.write("\n") + for idx, item in enumerate(q.all()): + if verbose: + sys.stdout.write(" * {}.{}: {}/{}\r".format(app_name, model_name, + idx + 1, ln)) + sys.stdout.flush() q = item.documents.filter( image__isnull=False).exclude(image='') if not q.count(): @@ -114,7 +122,10 @@ def migrate_main_image(apps, app_name, model_name): # by default get the lowest pk item.main_image = q.order_by('pk').all()[0] item.skip_history_when_saving = True + item._no_move = True item.save() + if verbose: + sys.stdout.write("\n") def m2m_historization_init(obj): |
