diff options
Diffstat (limited to 'ishtar_common')
14 files changed, 611 insertions, 8 deletions
| diff --git a/ishtar_common/management/commands/media_clean_unused.py b/ishtar_common/management/commands/media_clean_unused.py new file mode 100644 index 000000000..33237c1c7 --- /dev/null +++ b/ishtar_common/management/commands/media_clean_unused.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- + +""" +Copyright (c) 2014 Andrey Kolpakov - MIT License (MIT) +https://github.com/akolpakov/django-unused-media +""" + +import os + +import six.moves +from django.conf import settings +from django.core.management.base import BaseCommand + +from ishtar_common.utils import get_unused_media +from ishtar_common.utils import remove_empty_dirs + + +class Command(BaseCommand): + +    help = "Clean unused media files which have no reference in models" + +    # verbosity +    # 0 means minimal output +    # 1 means normal output (default). +    # 2 means verbose output + +    verbosity = 1 + +    def add_arguments(self, parser): +        parser.add_argument('--noinput', '--no-input', +                            dest='interactive', +                            action='store_false', +                            default=True, +                            help='Do not ask confirmation') + +        parser.add_argument('-e', '--exclude', +                            dest='exclude', +                            action='append', +                            default=[], +                            help='Exclude files by mask (only * is supported), can use multiple --exclude') + +        parser.add_argument('--remove-empty-dirs', +                            dest='remove_empty_dirs', +                            action='store_false', +                            default=False, +                            help='Remove empty dirs after files cleanup') + +        parser.add_argument('-n', '--dry-run', +                            dest='dry_run', +                            action='store_true', +                            default=False, +                            help='Dry run without any affect on your data') + +    def info(self, message): +        if self.verbosity >= 0: +            self.stdout.write(message) + +    def debug(self, message): +        if self.verbosity >= 1: +            self.stdout.write(message) + +    def _show_files_to_delete(self, unused_media): +        self.debug('Files to remove:') + +        for f in unused_media: +            self.debug(f) + +        self.info('Total files will be removed: {}'.format(len(unused_media))) + +    def handle(self, *args, **options): + +        if 'verbosity' in options: +            self.verbosity = options['verbosity'] + +        unused_media = get_unused_media(options.get('exclude') or []) + +        if not unused_media: +            self.info('Nothing to delete. Exit') +            return + +        if options.get('dry_run'): +            self._show_files_to_delete(unused_media) +            self.info('Dry run. Exit.') +            return + +        if options.get('interactive'): +            self._show_files_to_delete(unused_media) + +            # ask user +            question = 'Are you sure you want to remove {} unused files? (y/N)'.format(len(unused_media)) + +            if six.moves.input(question).upper() != 'Y': +                self.info('Interrupted by user. Exit.') +                return + +        for f in unused_media: +            self.debug('Remove %s' % f) +            os.remove(os.path.join(settings.MEDIA_ROOT, f)) + +        if options.get('remove_empty_dirs'): +            remove_empty_dirs() + +        self.info('Done. Total files removed: {}'.format(len(unused_media))) diff --git a/ishtar_common/management/commands/media_find_missing_files.py b/ishtar_common/management/commands/media_find_missing_files.py new file mode 100644 index 000000000..226699842 --- /dev/null +++ b/ishtar_common/management/commands/media_find_missing_files.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (C) 2019  Étienne Loks  <etienne.loks_AT_peacefrogsDOTnet> + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program.  If not, see <http://www.gnu.org/licenses/>. + +# See the file COPYING for details. + +import os.path +import sys + +from ishtar_common.utils import get_used_media, try_fix_file + +from django.core.management.base import BaseCommand + + +class Command(BaseCommand): +    args = '' +    help = 'Find missing files in media.' + +    def add_arguments(self, parser): +        parser.add_argument( +            '--exclude', nargs='?', dest='exclude', default=None, +            help="Field to exclude separated with \",\". Example: " +            "ishtar_common.Import.imported_file," +            "ishtar_common.Import.imported_images" +        ) +        parser.add_argument( +            '--limit', nargs='?', dest='limit', default=None, +            help="Field to limit to separated with \",\"." +        ) +        parser.add_argument( +            '--try-fix', dest='try-fix', action='store_true', +            default=False, +            help='Try to find file with similar names and copy the file.') +        parser.add_argument( +            '--find-fix', dest='find-fix', action='store_true', +            default=False, +            help='Try to find file with similar names and print them.') + +    def handle(self, *args, **options): +        exclude = options['exclude'].split(',') if options['exclude'] else [] +        limit = options['limit'].split(',') if options['limit'] else [] +        try_fix = options['try-fix'] +        find_fix = options['find-fix'] +        if try_fix and find_fix: +            self.stdout.write("try-fix and find-fix options are not " +                              "compatible.\n") +            return + +        missing = [] +        for media in get_used_media(exclude=exclude, limit=limit): +            if not os.path.isfile(media): +                missing.append(media) + +        if try_fix or find_fix: +            if find_fix: +                self.stdout.write("* potential similar file:\n") +            else: +                self.stdout.write("* fixes files:\n") +            for item in missing[:]: +                source_file = try_fix_file(item, make_copy=try_fix) +                if source_file: +                    missing.pop(missing.index(item)) +                    sys.stdout.write( +                        "{} <- {}\n".format(item.encode('utf-8'), +                                            source_file.encode('utf-8'))) + +        if missing: +            if find_fix or try_fix: +                self.stdout.write("* missing file with no similar file " +                                  "found:\n") +            for item in missing: +                sys.stdout.write(item.encode('utf-8') + "\n") +        else: +            self.stdout.write("No missing files.\n") diff --git a/ishtar_common/management/commands/media_simplify_filenames.py b/ishtar_common/management/commands/media_simplify_filenames.py new file mode 100644 index 000000000..2295431cd --- /dev/null +++ b/ishtar_common/management/commands/media_simplify_filenames.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (C) 2019  Étienne Loks  <etienne.loks_AT_peacefrogsDOTnet> + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program.  If not, see <http://www.gnu.org/licenses/>. + +# See the file COPYING for details. + +import os.path +import sys + + +from django.conf import settings +from django.core.management.base import BaseCommand + +from ishtar_common.utils import find_all_symlink, \ +    rename_and_simplify_media_name, get_used_media + + +class Command(BaseCommand): +    args = '' +    help = 'Simplify name of files in media' + +    def add_arguments(self, parser): +        parser.add_argument( +            '--exclude', nargs='?', dest='exclude', default=None, +            help="Field to exclude separated with \",\". Example: " +                 "ishtar_common.Import.imported_file," +                 "ishtar_common.Import.imported_images" +        ) +        parser.add_argument( +            '--limit', nargs='?', dest='limit', default=None, +            help="Field to limit to separated with \",\"." +        ) + +    def handle(self, *args, **options): +        exclude = options['exclude'].split(',') if options['exclude'] else [] +        limit = options['limit'].split(',') if options['limit'] else [] +        verbose = options['verbosity'] + +        current_links = dict(list(find_all_symlink(settings.MEDIA_ROOT))) + +        new_names = {} +        if verbose == 2: +            sys.stdout.write("* list current media...\n") +            sys.stdout.flush() + +        media = get_used_media(exclude=exclude, limit=limit, +                               return_object_and_field=True) +        ln = len(media) +        for idx, result in enumerate(sorted(media, key=lambda x: x[2])): +            if verbose == 2: +                nb_point = 10 +                sys.stdout.write("* processing {}/{} {}{}\r".format( +                    idx + 1, ln, (idx % nb_point) * ".", " " * nb_point) +                ) +                sys.stdout.flush() +            obj, field_name, media = result +            if not os.path.isfile(media): +                # missing file... +                continue +            if media not in new_names: +                new_name, modified = rename_and_simplify_media_name(media) +                if not modified: +                    continue +                new_names[media] = new_name +            setattr(obj, field_name, new_names[media]) +            obj._no_move = True +            obj._cached_label_checked = True +            obj.skip_history_when_saving = True +            obj.save() +            if verbose > 2: +                sys.stdout.write("{} renamed {}\n".format( +                    media, new_names[media].split(os.sep)[-1])) +            if media in current_links.keys(): +                os.remove(current_links[media]) +                os.symlink(new_names[media], current_links[media]) +                if verbose > 2: +                    sys.stdout.write( +                        "symlink {} changed to point to {}\n".format( +                            current_links[media], new_names[media])) +                current_links.pop(media) +        if verbose == 2: +            sys.stdout.write("\n") diff --git a/ishtar_common/migrations/0081_recreate_m2m_history.py b/ishtar_common/migrations/0081_recreate_m2m_history.py index e26a3f185..8f2779aed 100644 --- a/ishtar_common/migrations/0081_recreate_m2m_history.py +++ b/ishtar_common/migrations/0081_recreate_m2m_history.py @@ -2,6 +2,8 @@  # Generated by Django 1.11.10 on 2019-01-16 11:16  from __future__ import unicode_literals +import sys +  from django.db import migrations  from ishtar_common.utils_migrations import m2m_historization_init @@ -14,9 +16,17 @@ def recreate_m2m_migrations(apps, schema_editor):      history_models = [          ContextRecord, File, Find, Treatment, Operation, ArchaeologicalSite      ] +    sys.stdout.write("\n")      for model in history_models: -        for item in model.objects.all(): +        q = model.objects +        ln = q.count() +        for idx, item in enumerate(model.objects.all()): +            sys.stdout.write("{}: {}/{}\r".format(model, idx + 1, ln)) +            sys.stdout.flush()              m2m_historization_init(item) +        if ln: +            sys.stdout.write("\n") +    sys.stdout.write("\n")  class Migration(migrations.Migration): diff --git a/ishtar_common/migrations/0083_document_index_external_id.py b/ishtar_common/migrations/0083_document_index_external_id.py index 593962885..4f1dafcf9 100644 --- a/ishtar_common/migrations/0083_document_index_external_id.py +++ b/ishtar_common/migrations/0083_document_index_external_id.py @@ -1,13 +1,19 @@  # -*- coding: utf-8 -*-  # Generated by Django 1.11.10 on 2019-01-18 17:51  from __future__ import unicode_literals +import sys  from django.db import migrations  def gen_index(apps, schema_editor):      from ishtar_common.models import Document -    for doc in Document.objects.all(): +    sys.stdout.write("\n") +    q = Document.objects +    ln = q.count() +    for idx, doc in enumerate(Document.objects.all()): +        sys.stdout.write("  * {}/{}\r".format(idx, ln)) +        sys.stdout.flush()          doc._no_move = True          doc.save() diff --git a/ishtar_common/models.py b/ishtar_common/models.py index 425ee2a8d..5db454b14 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -4127,9 +4127,10 @@ class Document(BaseHistorizedItem, OwnPerms, ImageModel):                                    self._get_thumb_name(reference_path))                          self.image.name = reference_path[                                            len(settings.MEDIA_ROOT):] +                        self._no_move = True                          self.save(no_path_change=True)                      except OSError: -                        # file probably not on harddrive - will be cleaned +                        # file probably not on HDD - will be cleaned                          pass                      continue                  # create a link diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py index cf9f599c4..f51f9736a 100644 --- a/ishtar_common/tests.py +++ b/ishtar_common/tests.py @@ -45,7 +45,8 @@ from django.test.runner import DiscoverRunner  from ishtar_common import models  from ishtar_common import views  from ishtar_common.apps import admin_site -from ishtar_common.utils import post_save_point, update_data, move_dict_data +from ishtar_common.utils import post_save_point, update_data, move_dict_data, \ +    rename_and_simplify_media_name, try_fix_file  COMMON_FIXTURES = [ @@ -1657,3 +1658,41 @@ class DashboardTest(TestCase):              self.assertEqual(                  response.status_code, 200,                  "Reaching dashboard for item: {} return an error.".format(url)) + + +class CleanMedia(TestCase): + +    def test_rename(self): +        test_names = [ +            (u"éofficier2-12-02-04.93_gvK3hAr-1_2m7zZPn-1_nKhh2S2-1_"\ +             u"ONmUhfD-1_ymA3gGJ-1_XzJyRx3-1_PhvRcO8-1-thumb_ZwWMKBd.jpg", +             u"éofficier2-12-02-04.93-thumb.jpg"), +            (u"a_ZwWMKBd.jpg", False),  # no rename because too short +            (u"hoplala_gvK3hAr_2m7zZPn_nKhh2S2_ZwWMKBd.jpg", +             u"hoplala_gvK3hAr_2m7zZPn_nKhh2S2.jpg",),  # stop before because +            # another file exists +        ] +        base_dir = os.sep.join([settings.ROOT_PATH, u"..", u"ishtar_common", +                                u"tests", u"rename"]) +        for name, expected in test_names: +            name = os.sep.join([base_dir, name]) +            new_name, modif = rename_and_simplify_media_name(name, rename=False) +            if expected: +                self.assertTrue(modif) +                self.assertEqual(new_name, os.sep.join([base_dir, expected])) +            else: +                self.assertFalse(modif) + +    def test_try_fix(self): +        test_names = [ +            (u"hoplala_gvK3hAr_2m7zZPn_nKhh2S2_ZwWMKBd_ZwWMKBd.jpg", +             # non existing file +             u"hoplala_gvK3hAr_2m7zZPn.jpg",), +        ] +        base_dir = os.sep.join([settings.ROOT_PATH, u"..", u"ishtar_common", +                                u"tests", u"rename"]) +        for name, expected in test_names: +            name = os.sep.join([base_dir, name]) + +            found = try_fix_file(name, make_copy=False) +            self.assertEqual(found, expected) diff --git a/ishtar_common/tests/rename/a_ZwWMKBd.jpg b/ishtar_common/tests/rename/a_ZwWMKBd.jpg new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/ishtar_common/tests/rename/a_ZwWMKBd.jpg diff --git a/ishtar_common/tests/rename/hoplala_gvK3hAr_2m7zZPn.jpg b/ishtar_common/tests/rename/hoplala_gvK3hAr_2m7zZPn.jpg new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/ishtar_common/tests/rename/hoplala_gvK3hAr_2m7zZPn.jpg diff --git a/ishtar_common/tests/rename/hoplala_gvK3hAr_2m7zZPn_nKhh2S2_ZwWMKBd.jpg b/ishtar_common/tests/rename/hoplala_gvK3hAr_2m7zZPn_nKhh2S2_ZwWMKBd.jpg new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/ishtar_common/tests/rename/hoplala_gvK3hAr_2m7zZPn_nKhh2S2_ZwWMKBd.jpg diff --git a/ishtar_common/tests/rename/éofficier2-12-02-04.93_gvK3hAr-1_2m7zZPn-1_nKhh2S2-1_ONmUhfD-1-thumb.jpg b/ishtar_common/tests/rename/éofficier2-12-02-04.93_gvK3hAr-1_2m7zZPn-1_nKhh2S2-1_ONmUhfD-1-thumb.jpg new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/ishtar_common/tests/rename/éofficier2-12-02-04.93_gvK3hAr-1_2m7zZPn-1_nKhh2S2-1_ONmUhfD-1-thumb.jpg diff --git a/ishtar_common/tests/rename/éofficier2-12-02-04.93_gvK3hAr-1_2m7zZPn-1_nKhh2S2-1_ONmUhfD-1_ymA3gGJ-1_XzJyRx3-1_PhvRcO8-1-thumb_ZwWMKBd.jpg b/ishtar_common/tests/rename/éofficier2-12-02-04.93_gvK3hAr-1_2m7zZPn-1_nKhh2S2-1_ONmUhfD-1_ymA3gGJ-1_XzJyRx3-1_PhvRcO8-1-thumb_ZwWMKBd.jpg new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/ishtar_common/tests/rename/éofficier2-12-02-04.93_gvK3hAr-1_2m7zZPn-1_nKhh2S2-1_ONmUhfD-1_ymA3gGJ-1_XzJyRx3-1_PhvRcO8-1-thumb_ZwWMKBd.jpg diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py index f2fe34631..2156d4f95 100644 --- a/ishtar_common/utils.py +++ b/ishtar_common/utils.py @@ -24,12 +24,16 @@ from itertools import chain  import hashlib  import os  import random +import re  import requests  import shutil +import six  import subprocess +import sys  import tempfile  from django import forms +from django.apps import apps  from django.conf import settings  from django.conf.urls import url  from django.contrib.contenttypes.models import ContentType @@ -37,7 +41,9 @@ from django.contrib.gis.geos import GEOSGeometry  from django.contrib.sessions.backends.db import SessionStore  from django.core.cache import cache  from django.core.files import File +from django.core.validators import EMPTY_VALUES  from django.core.urlresolvers import reverse +from django.db import models  from django.http import HttpResponseRedirect  from django.utils.datastructures import MultiValueDict as BaseMultiValueDict  from django.utils.safestring import mark_safe @@ -966,3 +972,248 @@ def max_size_help():          settings.MAX_UPLOAD_SIZE      )      return msg + + +def find_all_symlink(dirname): +    for name in os.listdir(dirname): +        if name not in (os.curdir, os.pardir): +            full = os.path.join(dirname, name) +            if os.path.islink(full): +                yield full, os.readlink(full) + + +MEDIA_RE = [ +    re.compile(r"_[a-zA-Z0-9]{7}\-[0-9]"), +    re.compile(r"_[a-zA-Z0-9]{7}"), +] + + +def simplify_name(full_path_name, check_existing=False): +    """ +    Simplify a file name by removing auto save suffixes + +    :param full_path_name: full path name +    :param check_existing: prevent to give name of an existing file +    :return: +    """ +    name_exp = full_path_name.split(os.sep) +    path = os.sep.join(name_exp[0:-1]) +    name = name_exp[-1] +    current_name = name[:] +    ext = "" +    if u"." in name:  # remove extension if have one +        names = name.split(u".") +        name = u".".join(names[0:-1]) +        ext = u"." + names[-1] + +    while u"_" in name and len(name) > 15: +        oldname = name[:] +        for regex in MEDIA_RE: +            match = None +            for m in regex.finditer(name):  # get the last match +                match = m +            if match: +                new_name = name.replace(match.group(), '') +                full_new_name = os.sep.join([path, new_name + ext]) +                if not check_existing or not os.path.isfile(full_new_name): +                    # do not take the place of another file +                    name = new_name[:] +                    break +        if oldname == name: +            break +    return path, current_name, name + ext + + +def rename_and_simplify_media_name(full_path_name, rename=True): +    """ +    Simplify the name if possible +    :param full_path_name: full path name +    :param rename: rename file if True (default: True) +    :return: new full path name (or old if not changed), modified +    """ +    if not os.path.exists(full_path_name) or not os.path.isfile(full_path_name): +        return full_path_name, False + +    path, current_name, name = simplify_name(full_path_name, +                                             check_existing=True) +    if current_name == name: +        return full_path_name, False + +    full_new_name = os.sep.join([path, name]) +    if rename: +        os.rename(full_path_name, full_new_name) +    return full_new_name, True + + +def get_file_fields(): +    """ +    Get all fields which are inherited from FileField +    """ +    all_models = apps.get_models() +    fields = [] +    for model in all_models: +        for field in model._meta.get_fields(): +            if isinstance(field, models.FileField): +                fields.append(field) +    return fields + + +def get_used_media(exclude=None, limit=None, +                   return_object_and_field=False, debug=False): +    """ +    Get media which are still used in models + +    :param exclude: exclude fields, ex: ['ishtar_common.Import.imported_file', +    'ishtar_common.Import.imported_images'] +    :param limit: limit to some fields +    :param return_object_and_field: return associated object and field name +    :return: list of media filename or if return_object_and_field is set to +    True return (object, file field name, media filename) +    """ + +    if return_object_and_field: +        media = [] +    else: +        media = set() +    for field in get_file_fields(): +        if exclude and unicode(field) in exclude: +            continue +        if limit and unicode(field) not in limit: +            continue +        is_null = {'%s__isnull' % field.name: True} +        is_empty = {'%s' % field.name: ''} + +        storage = field.storage +        if debug: +            print("") +        q = field.model.objects.values('id', field.name)\ +            .exclude(**is_empty).exclude(**is_null) +        ln = q.count() +        for idx, res in enumerate(q): +            value = res[field.name] +            if debug: +                sys.stdout.write("* get_used_media {}: {}/{}\r".format( +                    field, idx, ln)) +                sys.stdout.flush() +            if value not in EMPTY_VALUES: +                if return_object_and_field: +                    media.append(( +                        field.model.objects.get(pk=res['id']), +                        field.name, +                        storage.path(value) +                    )) +                else: +                    media.add(storage.path(value)) +    return media + + +def get_all_media(exclude=None, debug=False): +    """ +    Get all media from MEDIA_ROOT +    """ + +    if not exclude: +        exclude = [] + +    media = set() +    full_dirs = list(os.walk(six.text_type(settings.MEDIA_ROOT))) +    ln_full = len(full_dirs) +    for idx_main, full_dir in enumerate(full_dirs): +        root, dirs, files = full_dir +        ln = len(files) +        if debug: +            print("") +        for idx, name in enumerate(files): +            if debug: +                sys.stdout.write("* get_all_media {} ({}/{}): {}/{}\r".format( +                    root.encode('utf-8'), idx_main, ln_full, idx, ln)) +                sys.stdout.flush() +            path = os.path.abspath(os.path.join(root, name)) +            relpath = os.path.relpath(path, settings.MEDIA_ROOT) +            in_exclude = False +            for e in exclude: +                if re.match(r'^%s$' % re.escape(e).replace('\\*', '.*'), +                            relpath): +                    in_exclude = True +                    break + +            if not in_exclude: +                media.add(path) +        else: +            if debug: +                sys.stdout.write("* get_all_media {} ({}/{})\r".format( +                    root.encode('utf-8'), idx_main, ln_full)) +    return media + + +def get_unused_media(exclude=None): +    """ +    Get media which are not used in models +    """ + +    if not exclude: +        exclude = [] + +    all_media = get_all_media(exclude) +    used_media = get_used_media() + +    return [x for x in all_media if x not in used_media] + + +def remove_unused_media(): +    """ +    Remove unused media +    """ +    remove_media(get_unused_media()) + + +def remove_media(files): +    """ +    Delete file from media dir +    """ +    for filename in files: +        os.remove(os.path.join(settings.MEDIA_ROOT, filename)) + + +def remove_empty_dirs(path=None): +    """ +    Recursively delete empty directories; return True if everything was deleted. +    """ + +    if not path: +        path = settings.MEDIA_ROOT + +    if not os.path.isdir(path): +        return False + +    listdir = [os.path.join(path, filename) for filename in os.listdir(path)] + +    if all(list(map(remove_empty_dirs, listdir))): +        os.rmdir(path) +        return True +    else: +        return False + + +def try_fix_file(filename, make_copy=True): +    """ +    Try to find a file with a similar name on the same dir. + +    :param filename: filename (full path) +    :param make_copy: make the copy of the similar file found +    :return: name of the similar file found or None +    """ +    path, current_name, simplified_ref_name = simplify_name( +        filename, check_existing=False) + +    # check existing files in the path +    for file in sorted(list(os.listdir(path))): +        full_file = os.sep.join([path, file]) +        if not os.path.isfile(full_file):  # must be a file +            continue +        _, _, name = simplify_name(full_file, check_existing=False) +        if simplified_ref_name.lower() == name.lower(): +            # a candidate is found +            if make_copy: +                shutil.copy2(full_file, filename) +            return file diff --git a/ishtar_common/utils_migrations.py b/ishtar_common/utils_migrations.py index 431fe5bd7..f51465262 100644 --- a/ishtar_common/utils_migrations.py +++ b/ishtar_common/utils_migrations.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals  import datetime  import json  import os +import sys  from django.core.files import File  from django.db import connection @@ -102,11 +103,18 @@ def reinit_last_modified(apps, app_name, models):              item.save() -def migrate_main_image(apps, app_name, model_name): +def migrate_main_image(apps, app_name, model_name, verbose=False):      model = apps.get_model(app_name, model_name) -    for item in model.objects.filter( -            documents__image__isnull=False).exclude( -            main_image__isnull=False).all(): +    q = model.objects.filter(documents__image__isnull=False).exclude( +        main_image__isnull=False) +    ln = q.count() +    if verbose: +        sys.stdout.write("\n") +    for idx, item in enumerate(q.all()): +        if verbose: +            sys.stdout.write("  * {}.{}: {}/{}\r".format(app_name, model_name, +                                                         idx + 1, ln)) +            sys.stdout.flush()          q = item.documents.filter(              image__isnull=False).exclude(image='')          if not q.count(): @@ -114,7 +122,10 @@ def migrate_main_image(apps, app_name, model_name):          # by default get the lowest pk          item.main_image = q.order_by('pk').all()[0]          item.skip_history_when_saving = True +        item._no_move = True          item.save() +    if verbose: +        sys.stdout.write("\n")  def m2m_historization_init(obj): | 
