Tools to manage media files.

* clean unused
* find missing
* rename and simplify

3 files changed, 285 insertions, 0 deletions

diff --git a/ishtar_common/management/commands/media_clean_unused.py b/ishtar_common/management/commands/media_clean_unused.py
new file mode 100644
index 000000000..33237c1c7
--- /dev/null
+++ b/ishtar_common/management/commands/media_clean_unused.py
@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+
+"""
+Copyright (c) 2014 Andrey Kolpakov - MIT License (MIT)
+https://github.com/akolpakov/django-unused-media
+"""
+
+import os
+
+import six.moves
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from ishtar_common.utils import get_unused_media
+from ishtar_common.utils import remove_empty_dirs
+
+
+class Command(BaseCommand):
+
+    help = "Clean unused media files which have no reference in models"
+
+    # verbosity
+    # 0 means minimal output
+    # 1 means normal output (default).
+    # 2 means verbose output
+
+    verbosity = 1
+
+    def add_arguments(self, parser):
+        parser.add_argument('--noinput', '--no-input',
+                            dest='interactive',
+                            action='store_false',
+                            default=True,
+                            help='Do not ask confirmation')
+
+        parser.add_argument('-e', '--exclude',
+                            dest='exclude',
+                            action='append',
+                            default=[],
+                            help='Exclude files by mask (only * is supported), can use multiple --exclude')
+
+        parser.add_argument('--remove-empty-dirs',
+                            dest='remove_empty_dirs',
+                            action='store_false',
+                            default=False,
+                            help='Remove empty dirs after files cleanup')
+
+        parser.add_argument('-n', '--dry-run',
+                            dest='dry_run',
+                            action='store_true',
+                            default=False,
+                            help='Dry run without any affect on your data')
+
+    def info(self, message):
+        if self.verbosity >= 0:
+            self.stdout.write(message)
+
+    def debug(self, message):
+        if self.verbosity >= 1:
+            self.stdout.write(message)
+
+    def _show_files_to_delete(self, unused_media):
+        self.debug('Files to remove:')
+
+        for f in unused_media:
+            self.debug(f)
+
+        self.info('Total files will be removed: {}'.format(len(unused_media)))
+
+    def handle(self, *args, **options):
+
+        if 'verbosity' in options:
+            self.verbosity = options['verbosity']
+
+        unused_media = get_unused_media(options.get('exclude') or [])
+
+        if not unused_media:
+            self.info('Nothing to delete. Exit')
+            return
+
+        if options.get('dry_run'):
+            self._show_files_to_delete(unused_media)
+            self.info('Dry run. Exit.')
+            return
+
+        if options.get('interactive'):
+            self._show_files_to_delete(unused_media)
+
+            # ask user
+            question = 'Are you sure you want to remove {} unused files? (y/N)'.format(len(unused_media))
+
+            if six.moves.input(question).upper() != 'Y':
+                self.info('Interrupted by user. Exit.')
+                return
+
+        for f in unused_media:
+            self.debug('Remove %s' % f)
+            os.remove(os.path.join(settings.MEDIA_ROOT, f))
+
+        if options.get('remove_empty_dirs'):
+            remove_empty_dirs()
+
+        self.info('Done. Total files removed: {}'.format(len(unused_media)))
diff --git a/ishtar_common/management/commands/media_find_missing_files.py b/ishtar_common/management/commands/media_find_missing_files.py
new file mode 100644
index 000000000..226699842
--- /dev/null
+++ b/ishtar_common/management/commands/media_find_missing_files.py
@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2019  Étienne Loks  <etienne.loks_AT_peacefrogsDOTnet>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# See the file COPYING for details.
+
+import os.path
+import sys
+
+from ishtar_common.utils import get_used_media, try_fix_file
+
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+    args = ''
+    help = 'Find missing files in media.'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--exclude', nargs='?', dest='exclude', default=None,
+            help="Field to exclude separated with \",\". Example: "
+            "ishtar_common.Import.imported_file,"
+            "ishtar_common.Import.imported_images"
+        )
+        parser.add_argument(
+            '--limit', nargs='?', dest='limit', default=None,
+            help="Field to limit to separated with \",\"."
+        )
+        parser.add_argument(
+            '--try-fix', dest='try-fix', action='store_true',
+            default=False,
+            help='Try to find file with similar names and copy the file.')
+        parser.add_argument(
+            '--find-fix', dest='find-fix', action='store_true',
+            default=False,
+            help='Try to find file with similar names and print them.')
+
+    def handle(self, *args, **options):
+        exclude = options['exclude'].split(',') if options['exclude'] else []
+        limit = options['limit'].split(',') if options['limit'] else []
+        try_fix = options['try-fix']
+        find_fix = options['find-fix']
+        if try_fix and find_fix:
+            self.stdout.write("try-fix and find-fix options are not "
+                              "compatible.\n")
+            return
+
+        missing = []
+        for media in get_used_media(exclude=exclude, limit=limit):
+            if not os.path.isfile(media):
+                missing.append(media)
+
+        if try_fix or find_fix:
+            if find_fix:
+                self.stdout.write("* potential similar file:\n")
+            else:
+                self.stdout.write("* fixes files:\n")
+            for item in missing[:]:
+                source_file = try_fix_file(item, make_copy=try_fix)
+                if source_file:
+                    missing.pop(missing.index(item))
+                    sys.stdout.write(
+                        "{} <- {}\n".format(item.encode('utf-8'),
+                                            source_file.encode('utf-8')))
+
+        if missing:
+            if find_fix or try_fix:
+                self.stdout.write("* missing file with no similar file "
+                                  "found:\n")
+            for item in missing:
+                sys.stdout.write(item.encode('utf-8') + "\n")
+        else:
+            self.stdout.write("No missing files.\n")
diff --git a/ishtar_common/management/commands/media_simplify_filenames.py b/ishtar_common/management/commands/media_simplify_filenames.py
new file mode 100644
index 000000000..2295431cd
--- /dev/null
+++ b/ishtar_common/management/commands/media_simplify_filenames.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2019  Étienne Loks  <etienne.loks_AT_peacefrogsDOTnet>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# See the file COPYING for details.
+
+import os.path
+import sys
+
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from ishtar_common.utils import find_all_symlink, \
+    rename_and_simplify_media_name, get_used_media
+
+
+class Command(BaseCommand):
+    args = ''
+    help = 'Simplify name of files in media'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--exclude', nargs='?', dest='exclude', default=None,
+            help="Field to exclude separated with \",\". Example: "
+                 "ishtar_common.Import.imported_file,"
+                 "ishtar_common.Import.imported_images"
+        )
+        parser.add_argument(
+            '--limit', nargs='?', dest='limit', default=None,
+            help="Field to limit to separated with \",\"."
+        )
+
+    def handle(self, *args, **options):
+        exclude = options['exclude'].split(',') if options['exclude'] else []
+        limit = options['limit'].split(',') if options['limit'] else []
+        verbose = options['verbosity']
+
+        current_links = dict(list(find_all_symlink(settings.MEDIA_ROOT)))
+
+        new_names = {}
+        if verbose == 2:
+            sys.stdout.write("* list current media...\n")
+            sys.stdout.flush()
+
+        media = get_used_media(exclude=exclude, limit=limit,
+                               return_object_and_field=True)
+        ln = len(media)
+        for idx, result in enumerate(sorted(media, key=lambda x: x[2])):
+            if verbose == 2:
+                nb_point = 10
+                sys.stdout.write("* processing {}/{} {}{}\r".format(
+                    idx + 1, ln, (idx % nb_point) * ".", " " * nb_point)
+                )
+                sys.stdout.flush()
+            obj, field_name, media = result
+            if not os.path.isfile(media):
+                # missing file...
+                continue
+            if media not in new_names:
+                new_name, modified = rename_and_simplify_media_name(media)
+                if not modified:
+                    continue
+                new_names[media] = new_name
+            setattr(obj, field_name, new_names[media])
+            obj._no_move = True
+            obj._cached_label_checked = True
+            obj.skip_history_when_saving = True
+            obj.save()
+            if verbose > 2:
+                sys.stdout.write("{} renamed {}\n".format(
+                    media, new_names[media].split(os.sep)[-1]))
+            if media in current_links.keys():
+                os.remove(current_links[media])
+                os.symlink(new_names[media], current_links[media])
+                if verbose > 2:
+                    sys.stdout.write(
+                        "symlink {} changed to point to {}\n".format(
+                            current_links[media], new_names[media]))
+                current_links.pop(media)
+        if verbose == 2:
+            sys.stdout.write("\n")