Tools to manage media files.

* clean unused
* find missing
* rename and simplify

15 files changed, 615 insertions, 11 deletions

diff --git a/archaeological_finds/migrations/0054_migrate_main_image.py b/archaeological_finds/migrations/0054_migrate_main_image.py
index 1a7f942c4..6d11a5a67 100644
--- a/archaeological_finds/migrations/0054_migrate_main_image.py
+++ b/archaeological_finds/migrations/0054_migrate_main_image.py
@@ -7,9 +7,10 @@ from ishtar_common.utils_migrations import migrate_main_image
 
 
 def migrate_main_image_script(apps, schema):
-    migrate_main_image(apps, 'archaeological_finds', 'Find')
-    migrate_main_image(apps, 'archaeological_finds', 'Treatment')
-    migrate_main_image(apps, 'archaeological_finds', 'TreatmentFile')
+    migrate_main_image(apps, 'archaeological_finds', 'Find', verbose=True)
+    migrate_main_image(apps, 'archaeological_finds', 'Treatment', verbose=True)
+    migrate_main_image(apps, 'archaeological_finds', 'TreatmentFile',
+                       verbose=True)
 
 
 class Migration(migrations.Migration):
diff --git a/ishtar_common/management/commands/media_clean_unused.py b/ishtar_common/management/commands/media_clean_unused.py
new file mode 100644
index 000000000..33237c1c7
--- /dev/null
+++ b/ishtar_common/management/commands/media_clean_unused.py
@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+
+"""
+Copyright (c) 2014 Andrey Kolpakov - MIT License (MIT)
+https://github.com/akolpakov/django-unused-media
+"""
+
+import os
+
+import six.moves
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from ishtar_common.utils import get_unused_media
+from ishtar_common.utils import remove_empty_dirs
+
+
+class Command(BaseCommand):
+
+    help = "Clean unused media files which have no reference in models"
+
+    # verbosity
+    # 0 means minimal output
+    # 1 means normal output (default).
+    # 2 means verbose output
+
+    verbosity = 1
+
+    def add_arguments(self, parser):
+        parser.add_argument('--noinput', '--no-input',
+                            dest='interactive',
+                            action='store_false',
+                            default=True,
+                            help='Do not ask confirmation')
+
+        parser.add_argument('-e', '--exclude',
+                            dest='exclude',
+                            action='append',
+                            default=[],
+                            help='Exclude files by mask (only * is supported), can use multiple --exclude')
+
+        parser.add_argument('--remove-empty-dirs',
+                            dest='remove_empty_dirs',
+                            action='store_false',
+                            default=False,
+                            help='Remove empty dirs after files cleanup')
+
+        parser.add_argument('-n', '--dry-run',
+                            dest='dry_run',
+                            action='store_true',
+                            default=False,
+                            help='Dry run without any affect on your data')
+
+    def info(self, message):
+        if self.verbosity >= 0:
+            self.stdout.write(message)
+
+    def debug(self, message):
+        if self.verbosity >= 1:
+            self.stdout.write(message)
+
+    def _show_files_to_delete(self, unused_media):
+        self.debug('Files to remove:')
+
+        for f in unused_media:
+            self.debug(f)
+
+        self.info('Total files will be removed: {}'.format(len(unused_media)))
+
+    def handle(self, *args, **options):
+
+        if 'verbosity' in options:
+            self.verbosity = options['verbosity']
+
+        unused_media = get_unused_media(options.get('exclude') or [])
+
+        if not unused_media:
+            self.info('Nothing to delete. Exit')
+            return
+
+        if options.get('dry_run'):
+            self._show_files_to_delete(unused_media)
+            self.info('Dry run. Exit.')
+            return
+
+        if options.get('interactive'):
+            self._show_files_to_delete(unused_media)
+
+            # ask user
+            question = 'Are you sure you want to remove {} unused files? (y/N)'.format(len(unused_media))
+
+            if six.moves.input(question).upper() != 'Y':
+                self.info('Interrupted by user. Exit.')
+                return
+
+        for f in unused_media:
+            self.debug('Remove %s' % f)
+            os.remove(os.path.join(settings.MEDIA_ROOT, f))
+
+        if options.get('remove_empty_dirs'):
+            remove_empty_dirs()
+
+        self.info('Done. Total files removed: {}'.format(len(unused_media)))
diff --git a/ishtar_common/management/commands/media_find_missing_files.py b/ishtar_common/management/commands/media_find_missing_files.py
new file mode 100644
index 000000000..226699842
--- /dev/null
+++ b/ishtar_common/management/commands/media_find_missing_files.py
@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2019  Étienne Loks  <etienne.loks_AT_peacefrogsDOTnet>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# See the file COPYING for details.
+
+import os.path
+import sys
+
+from ishtar_common.utils import get_used_media, try_fix_file
+
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+    args = ''
+    help = 'Find missing files in media.'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--exclude', nargs='?', dest='exclude', default=None,
+            help="Field to exclude separated with \",\". Example: "
+            "ishtar_common.Import.imported_file,"
+            "ishtar_common.Import.imported_images"
+        )
+        parser.add_argument(
+            '--limit', nargs='?', dest='limit', default=None,
+            help="Field to limit to separated with \",\"."
+        )
+        parser.add_argument(
+            '--try-fix', dest='try-fix', action='store_true',
+            default=False,
+            help='Try to find file with similar names and copy the file.')
+        parser.add_argument(
+            '--find-fix', dest='find-fix', action='store_true',
+            default=False,
+            help='Try to find file with similar names and print them.')
+
+    def handle(self, *args, **options):
+        exclude = options['exclude'].split(',') if options['exclude'] else []
+        limit = options['limit'].split(',') if options['limit'] else []
+        try_fix = options['try-fix']
+        find_fix = options['find-fix']
+        if try_fix and find_fix:
+            self.stdout.write("try-fix and find-fix options are not "
+                              "compatible.\n")
+            return
+
+        missing = []
+        for media in get_used_media(exclude=exclude, limit=limit):
+            if not os.path.isfile(media):
+                missing.append(media)
+
+        if try_fix or find_fix:
+            if find_fix:
+                self.stdout.write("* potential similar file:\n")
+            else:
+                self.stdout.write("* fixes files:\n")
+            for item in missing[:]:
+                source_file = try_fix_file(item, make_copy=try_fix)
+                if source_file:
+                    missing.pop(missing.index(item))
+                    sys.stdout.write(
+                        "{} <- {}\n".format(item.encode('utf-8'),
+                                            source_file.encode('utf-8')))
+
+        if missing:
+            if find_fix or try_fix:
+                self.stdout.write("* missing file with no similar file "
+                                  "found:\n")
+            for item in missing:
+                sys.stdout.write(item.encode('utf-8') + "\n")
+        else:
+            self.stdout.write("No missing files.\n")
diff --git a/ishtar_common/management/commands/media_simplify_filenames.py b/ishtar_common/management/commands/media_simplify_filenames.py
new file mode 100644
index 000000000..2295431cd
--- /dev/null
+++ b/ishtar_common/management/commands/media_simplify_filenames.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2019  Étienne Loks  <etienne.loks_AT_peacefrogsDOTnet>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# See the file COPYING for details.
+
+import os.path
+import sys
+
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from ishtar_common.utils import find_all_symlink, \
+    rename_and_simplify_media_name, get_used_media
+
+
+class Command(BaseCommand):
+    args = ''
+    help = 'Simplify name of files in media'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--exclude', nargs='?', dest='exclude', default=None,
+            help="Field to exclude separated with \",\". Example: "
+                 "ishtar_common.Import.imported_file,"
+                 "ishtar_common.Import.imported_images"
+        )
+        parser.add_argument(
+            '--limit', nargs='?', dest='limit', default=None,
+            help="Field to limit to separated with \",\"."
+        )
+
+    def handle(self, *args, **options):
+        exclude = options['exclude'].split(',') if options['exclude'] else []
+        limit = options['limit'].split(',') if options['limit'] else []
+        verbose = options['verbosity']
+
+        current_links = dict(list(find_all_symlink(settings.MEDIA_ROOT)))
+
+        new_names = {}
+        if verbose == 2:
+            sys.stdout.write("* list current media...\n")
+            sys.stdout.flush()
+
+        media = get_used_media(exclude=exclude, limit=limit,
+                               return_object_and_field=True)
+        ln = len(media)
+        for idx, result in enumerate(sorted(media, key=lambda x: x[2])):
+            if verbose == 2:
+                nb_point = 10
+                sys.stdout.write("* processing {}/{} {}{}\r".format(
+                    idx + 1, ln, (idx % nb_point) * ".", " " * nb_point)
+                )
+                sys.stdout.flush()
+            obj, field_name, media = result
+            if not os.path.isfile(media):
+                # missing file...
+                continue
+            if media not in new_names:
+                new_name, modified = rename_and_simplify_media_name(media)
+                if not modified:
+                    continue
+                new_names[media] = new_name
+            setattr(obj, field_name, new_names[media])
+            obj._no_move = True
+            obj._cached_label_checked = True
+            obj.skip_history_when_saving = True
+            obj.save()
+            if verbose > 2:
+                sys.stdout.write("{} renamed {}\n".format(
+                    media, new_names[media].split(os.sep)[-1]))
+            if media in current_links.keys():
+                os.remove(current_links[media])
+                os.symlink(new_names[media], current_links[media])
+                if verbose > 2:
+                    sys.stdout.write(
+                        "symlink {} changed to point to {}\n".format(
+                            current_links[media], new_names[media]))
+                current_links.pop(media)
+        if verbose == 2:
+            sys.stdout.write("\n")
diff --git a/ishtar_common/migrations/0081_recreate_m2m_history.py b/ishtar_common/migrations/0081_recreate_m2m_history.py
index e26a3f185..8f2779aed 100644
--- a/ishtar_common/migrations/0081_recreate_m2m_history.py
+++ b/ishtar_common/migrations/0081_recreate_m2m_history.py
@@ -2,6 +2,8 @@
 # Generated by Django 1.11.10 on 2019-01-16 11:16
 from __future__ import unicode_literals
 
+import sys
+
 from django.db import migrations
 from ishtar_common.utils_migrations import m2m_historization_init
 
@@ -14,9 +16,17 @@ def recreate_m2m_migrations(apps, schema_editor):
     history_models = [
         ContextRecord, File, Find, Treatment, Operation, ArchaeologicalSite
     ]
+    sys.stdout.write("\n")
     for model in history_models:
-        for item in model.objects.all():
+        q = model.objects
+        ln = q.count()
+        for idx, item in enumerate(model.objects.all()):
+            sys.stdout.write("{}: {}/{}\r".format(model, idx + 1, ln))
+            sys.stdout.flush()
             m2m_historization_init(item)
+        if ln:
+            sys.stdout.write("\n")
+    sys.stdout.write("\n")
 
 
 class Migration(migrations.Migration):
diff --git a/ishtar_common/migrations/0083_document_index_external_id.py b/ishtar_common/migrations/0083_document_index_external_id.py
index 593962885..4f1dafcf9 100644
--- a/ishtar_common/migrations/0083_document_index_external_id.py
+++ b/ishtar_common/migrations/0083_document_index_external_id.py
@@ -1,13 +1,19 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.11.10 on 2019-01-18 17:51
 from __future__ import unicode_literals
+import sys
 
 from django.db import migrations
 
 
 def gen_index(apps, schema_editor):
     from ishtar_common.models import Document
-    for doc in Document.objects.all():
+    sys.stdout.write("\n")
+    q = Document.objects
+    ln = q.count()
+    for idx, doc in enumerate(Document.objects.all()):
+        sys.stdout.write("  * {}/{}\r".format(idx, ln))
+        sys.stdout.flush()
         doc._no_move = True
         doc.save()
 
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index 425ee2a8d..5db454b14 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -4127,9 +4127,10 @@ class Document(BaseHistorizedItem, OwnPerms, ImageModel):
                                   self._get_thumb_name(reference_path))
                         self.image.name = reference_path[
                                           len(settings.MEDIA_ROOT):]
+                        self._no_move = True
                         self.save(no_path_change=True)
                     except OSError:
-                        # file probably not on harddrive - will be cleaned
+                        # file probably not on HDD - will be cleaned
                         pass
                     continue
                 # create a link
diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py
index cf9f599c4..f51f9736a 100644
--- a/ishtar_common/tests.py
+++ b/ishtar_common/tests.py
@@ -45,7 +45,8 @@ from django.test.runner import DiscoverRunner
 from ishtar_common import models
 from ishtar_common import views
 from ishtar_common.apps import admin_site
-from ishtar_common.utils import post_save_point, update_data, move_dict_data
+from ishtar_common.utils import post_save_point, update_data, move_dict_data, \
+    rename_and_simplify_media_name, try_fix_file
 
 
 COMMON_FIXTURES = [
@@ -1657,3 +1658,41 @@ class DashboardTest(TestCase):
             self.assertEqual(
                 response.status_code, 200,
                 "Reaching dashboard for item: {} return an error.".format(url))
+
+
+class CleanMedia(TestCase):
+
+    def test_rename(self):
+        test_names = [
+            (u"éofficier2-12-02-04.93_gvK3hAr-1_2m7zZPn-1_nKhh2S2-1_"\
+             u"ONmUhfD-1_ymA3gGJ-1_XzJyRx3-1_PhvRcO8-1-thumb_ZwWMKBd.jpg",
+             u"éofficier2-12-02-04.93-thumb.jpg"),
+            (u"a_ZwWMKBd.jpg", False),  # no rename because too short
+            (u"hoplala_gvK3hAr_2m7zZPn_nKhh2S2_ZwWMKBd.jpg",
+             u"hoplala_gvK3hAr_2m7zZPn_nKhh2S2.jpg",),  # stop before because
+            # another file exists
+        ]
+        base_dir = os.sep.join([settings.ROOT_PATH, u"..", u"ishtar_common",
+                                u"tests", u"rename"])
+        for name, expected in test_names:
+            name = os.sep.join([base_dir, name])
+            new_name, modif = rename_and_simplify_media_name(name, rename=False)
+            if expected:
+                self.assertTrue(modif)
+                self.assertEqual(new_name, os.sep.join([base_dir, expected]))
+            else:
+                self.assertFalse(modif)
+
+    def test_try_fix(self):
+        test_names = [
+            (u"hoplala_gvK3hAr_2m7zZPn_nKhh2S2_ZwWMKBd_ZwWMKBd.jpg",
+             # non existing file
+             u"hoplala_gvK3hAr_2m7zZPn.jpg",),
+        ]
+        base_dir = os.sep.join([settings.ROOT_PATH, u"..", u"ishtar_common",
+                                u"tests", u"rename"])
+        for name, expected in test_names:
+            name = os.sep.join([base_dir, name])
+
+            found = try_fix_file(name, make_copy=False)
+            self.assertEqual(found, expected)
diff --git a/example_project/media/imported/imported_files_here b/ishtar_common/tests/rename/a_ZwWMKBd.jpg
index e69de29bb..e69de29bb 100644
--- a/example_project/media/imported/imported_files_here
+++ b/ishtar_common/tests/rename/a_ZwWMKBd.jpg
diff --git a/ishtar_common/tests/rename/hoplala_gvK3hAr_2m7zZPn.jpg b/ishtar_common/tests/rename/hoplala_gvK3hAr_2m7zZPn.jpg
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/ishtar_common/tests/rename/hoplala_gvK3hAr_2m7zZPn.jpg
diff --git a/ishtar_common/tests/rename/hoplala_gvK3hAr_2m7zZPn_nKhh2S2_ZwWMKBd.jpg b/ishtar_common/tests/rename/hoplala_gvK3hAr_2m7zZPn_nKhh2S2_ZwWMKBd.jpg
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/ishtar_common/tests/rename/hoplala_gvK3hAr_2m7zZPn_nKhh2S2_ZwWMKBd.jpg
diff --git a/ishtar_common/tests/rename/éofficier2-12-02-04.93_gvK3hAr-1_2m7zZPn-1_nKhh2S2-1_ONmUhfD-1-thumb.jpg b/ishtar_common/tests/rename/éofficier2-12-02-04.93_gvK3hAr-1_2m7zZPn-1_nKhh2S2-1_ONmUhfD-1-thumb.jpg
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/ishtar_common/tests/rename/éofficier2-12-02-04.93_gvK3hAr-1_2m7zZPn-1_nKhh2S2-1_ONmUhfD-1-thumb.jpg
diff --git a/ishtar_common/tests/rename/éofficier2-12-02-04.93_gvK3hAr-1_2m7zZPn-1_nKhh2S2-1_ONmUhfD-1_ymA3gGJ-1_XzJyRx3-1_PhvRcO8-1-thumb_ZwWMKBd.jpg b/ishtar_common/tests/rename/éofficier2-12-02-04.93_gvK3hAr-1_2m7zZPn-1_nKhh2S2-1_ONmUhfD-1_ymA3gGJ-1_XzJyRx3-1_PhvRcO8-1-thumb_ZwWMKBd.jpg
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/ishtar_common/tests/rename/éofficier2-12-02-04.93_gvK3hAr-1_2m7zZPn-1_nKhh2S2-1_ONmUhfD-1_ymA3gGJ-1_XzJyRx3-1_PhvRcO8-1-thumb_ZwWMKBd.jpg
diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py
index f2fe34631..2156d4f95 100644
--- a/ishtar_common/utils.py
+++ b/ishtar_common/utils.py
@@ -24,12 +24,16 @@ from itertools import chain
 import hashlib
 import os
 import random
+import re
 import requests
 import shutil
+import six
 import subprocess
+import sys
 import tempfile
 
 from django import forms
+from django.apps import apps
 from django.conf import settings
 from django.conf.urls import url
 from django.contrib.contenttypes.models import ContentType
@@ -37,7 +41,9 @@ from django.contrib.gis.geos import GEOSGeometry
 from django.contrib.sessions.backends.db import SessionStore
 from django.core.cache import cache
 from django.core.files import File
+from django.core.validators import EMPTY_VALUES
 from django.core.urlresolvers import reverse
+from django.db import models
 from django.http import HttpResponseRedirect
 from django.utils.datastructures import MultiValueDict as BaseMultiValueDict
 from django.utils.safestring import mark_safe
@@ -966,3 +972,248 @@ def max_size_help():
         settings.MAX_UPLOAD_SIZE
     )
     return msg
+
+
+def find_all_symlink(dirname):
+    for name in os.listdir(dirname):
+        if name not in (os.curdir, os.pardir):
+            full = os.path.join(dirname, name)
+            if os.path.islink(full):
+                yield full, os.readlink(full)
+
+
+MEDIA_RE = [
+    re.compile(r"_[a-zA-Z0-9]{7}\-[0-9]"),
+    re.compile(r"_[a-zA-Z0-9]{7}"),
+]
+
+
+def simplify_name(full_path_name, check_existing=False):
+    """
+    Simplify a file name by removing auto save suffixes
+
+    :param full_path_name: full path name
+    :param check_existing: prevent to give name of an existing file
+    :return:
+    """
+    name_exp = full_path_name.split(os.sep)
+    path = os.sep.join(name_exp[0:-1])
+    name = name_exp[-1]
+    current_name = name[:]
+    ext = ""
+    if u"." in name:  # remove extension if have one
+        names = name.split(u".")
+        name = u".".join(names[0:-1])
+        ext = u"." + names[-1]
+
+    while u"_" in name and len(name) > 15:
+        oldname = name[:]
+        for regex in MEDIA_RE:
+            match = None
+            for m in regex.finditer(name):  # get the last match
+                match = m
+            if match:
+                new_name = name.replace(match.group(), '')
+                full_new_name = os.sep.join([path, new_name + ext])
+                if not check_existing or not os.path.isfile(full_new_name):
+                    # do not take the place of another file
+                    name = new_name[:]
+                    break
+        if oldname == name:
+            break
+    return path, current_name, name + ext
+
+
+def rename_and_simplify_media_name(full_path_name, rename=True):
+    """
+    Simplify the name if possible
+    :param full_path_name: full path name
+    :param rename: rename file if True (default: True)
+    :return: new full path name (or old if not changed), modified
+    """
+    if not os.path.exists(full_path_name) or not os.path.isfile(full_path_name):
+        return full_path_name, False
+
+    path, current_name, name = simplify_name(full_path_name,
+                                             check_existing=True)
+    if current_name == name:
+        return full_path_name, False
+
+    full_new_name = os.sep.join([path, name])
+    if rename:
+        os.rename(full_path_name, full_new_name)
+    return full_new_name, True
+
+
+def get_file_fields():
+    """
+    Get all fields which are inherited from FileField
+    """
+    all_models = apps.get_models()
+    fields = []
+    for model in all_models:
+        for field in model._meta.get_fields():
+            if isinstance(field, models.FileField):
+                fields.append(field)
+    return fields
+
+
+def get_used_media(exclude=None, limit=None,
+                   return_object_and_field=False, debug=False):
+    """
+    Get media which are still used in models
+
+    :param exclude: exclude fields, ex: ['ishtar_common.Import.imported_file',
+    'ishtar_common.Import.imported_images']
+    :param limit: limit to some fields
+    :param return_object_and_field: return associated object and field name
+    :return: list of media filename or if return_object_and_field is set to
+    True return (object, file field name, media filename)
+    """
+
+    if return_object_and_field:
+        media = []
+    else:
+        media = set()
+    for field in get_file_fields():
+        if exclude and unicode(field) in exclude:
+            continue
+        if limit and unicode(field) not in limit:
+            continue
+        is_null = {'%s__isnull' % field.name: True}
+        is_empty = {'%s' % field.name: ''}
+
+        storage = field.storage
+        if debug:
+            print("")
+        q = field.model.objects.values('id', field.name)\
+            .exclude(**is_empty).exclude(**is_null)
+        ln = q.count()
+        for idx, res in enumerate(q):
+            value = res[field.name]
+            if debug:
+                sys.stdout.write("* get_used_media {}: {}/{}\r".format(
+                    field, idx, ln))
+                sys.stdout.flush()
+            if value not in EMPTY_VALUES:
+                if return_object_and_field:
+                    media.append((
+                        field.model.objects.get(pk=res['id']),
+                        field.name,
+                        storage.path(value)
+                    ))
+                else:
+                    media.add(storage.path(value))
+    return media
+
+
+def get_all_media(exclude=None, debug=False):
+    """
+    Get all media from MEDIA_ROOT
+    """
+
+    if not exclude:
+        exclude = []
+
+    media = set()
+    full_dirs = list(os.walk(six.text_type(settings.MEDIA_ROOT)))
+    ln_full = len(full_dirs)
+    for idx_main, full_dir in enumerate(full_dirs):
+        root, dirs, files = full_dir
+        ln = len(files)
+        if debug:
+            print("")
+        for idx, name in enumerate(files):
+            if debug:
+                sys.stdout.write("* get_all_media {} ({}/{}): {}/{}\r".format(
+                    root.encode('utf-8'), idx_main, ln_full, idx, ln))
+                sys.stdout.flush()
+            path = os.path.abspath(os.path.join(root, name))
+            relpath = os.path.relpath(path, settings.MEDIA_ROOT)
+            in_exclude = False
+            for e in exclude:
+                if re.match(r'^%s$' % re.escape(e).replace('\\*', '.*'),
+                            relpath):
+                    in_exclude = True
+                    break
+
+            if not in_exclude:
+                media.add(path)
+        else:
+            if debug:
+                sys.stdout.write("* get_all_media {} ({}/{})\r".format(
+                    root.encode('utf-8'), idx_main, ln_full))
+    return media
+
+
+def get_unused_media(exclude=None):
+    """
+    Get media which are not used in models
+    """
+
+    if not exclude:
+        exclude = []
+
+    all_media = get_all_media(exclude)
+    used_media = get_used_media()
+
+    return [x for x in all_media if x not in used_media]
+
+
+def remove_unused_media():
+    """
+    Remove unused media
+    """
+    remove_media(get_unused_media())
+
+
+def remove_media(files):
+    """
+    Delete file from media dir
+    """
+    for filename in files:
+        os.remove(os.path.join(settings.MEDIA_ROOT, filename))
+
+
+def remove_empty_dirs(path=None):
+    """
+    Recursively delete empty directories; return True if everything was deleted.
+    """
+
+    if not path:
+        path = settings.MEDIA_ROOT
+
+    if not os.path.isdir(path):
+        return False
+
+    listdir = [os.path.join(path, filename) for filename in os.listdir(path)]
+
+    if all(list(map(remove_empty_dirs, listdir))):
+        os.rmdir(path)
+        return True
+    else:
+        return False
+
+
+def try_fix_file(filename, make_copy=True):
+    """
+    Try to find a file with a similar name on the same dir.
+
+    :param filename: filename (full path)
+    :param make_copy: make the copy of the similar file found
+    :return: name of the similar file found or None
+    """
+    path, current_name, simplified_ref_name = simplify_name(
+        filename, check_existing=False)
+
+    # check existing files in the path
+    for file in sorted(list(os.listdir(path))):
+        full_file = os.sep.join([path, file])
+        if not os.path.isfile(full_file):  # must be a file
+            continue
+        _, _, name = simplify_name(full_file, check_existing=False)
+        if simplified_ref_name.lower() == name.lower():
+            # a candidate is found
+            if make_copy:
+                shutil.copy2(full_file, filename)
+            return file
diff --git a/ishtar_common/utils_migrations.py b/ishtar_common/utils_migrations.py
index 431fe5bd7..f51465262 100644
--- a/ishtar_common/utils_migrations.py
+++ b/ishtar_common/utils_migrations.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import datetime
 import json
 import os
+import sys
 
 from django.core.files import File
 from django.db import connection
@@ -102,11 +103,18 @@ def reinit_last_modified(apps, app_name, models):
             item.save()
 
 
-def migrate_main_image(apps, app_name, model_name):
+def migrate_main_image(apps, app_name, model_name, verbose=False):
     model = apps.get_model(app_name, model_name)
-    for item in model.objects.filter(
-            documents__image__isnull=False).exclude(
-            main_image__isnull=False).all():
+    q = model.objects.filter(documents__image__isnull=False).exclude(
+        main_image__isnull=False)
+    ln = q.count()
+    if verbose:
+        sys.stdout.write("\n")
+    for idx, item in enumerate(q.all()):
+        if verbose:
+            sys.stdout.write("  * {}.{}: {}/{}\r".format(app_name, model_name,
+                                                         idx + 1, ln))
+            sys.stdout.flush()
         q = item.documents.filter(
             image__isnull=False).exclude(image='')
         if not q.count():
@@ -114,7 +122,10 @@ def migrate_main_image(apps, app_name, model_name):
         # by default get the lowest pk
         item.main_image = q.order_by('pk').all()[0]
         item.skip_history_when_saving = True
+        item._no_move = True
         item.save()
+    if verbose:
+        sys.stdout.write("\n")
 
 
 def m2m_historization_init(obj):