path: root/ishtar_common/utils.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2013-2016 Étienne Loks  <etienne.loks_AT_peacefrogsDOTnet>

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# See the file COPYING for details.

import datetime
from functools import wraps
from itertools import chain
import hashlib
import os
import random
import shutil
import subprocess
import tempfile

from django import forms
from django.conf import settings
from django.contrib.gis.geos import GEOSGeometry
from django.contrib.sessions.backends.db import SessionStore
from django.core.cache import cache
from django.core.exceptions import FieldDoesNotExist
from django.core.files import File
from django.core.urlresolvers import reverse
from django.utils.datastructures import MultiValueDict as BaseMultiValueDict
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _, ugettext
from django.template.defaultfilters import slugify


class BColors:
    """
    Bash colors. Don't forget to finish your colored string with ENDC.
    """
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'


class MultiValueDict(BaseMultiValueDict):
    def get(self, *args, **kwargs):
        v = super(MultiValueDict, self).getlist(*args, **kwargs)
        if callable(v):
            v = v()
        if type(v) in (list, tuple) and len(v) > 1:
            v = ",".join(v)
        elif type(v) not in (int, unicode):
            v = super(MultiValueDict, self).get(*args, **kwargs)
        return v

    def getlist(self, *args, **kwargs):
        lst = super(MultiValueDict, self).getlist(*args, **kwargs)
        if type(lst) not in (tuple, list):
            lst = [lst]
        return lst


def get_current_year():
    return datetime.datetime.now().year


def get_cache(cls, extra_args=tuple(), app_label=None):
    if not app_label:
        app_label = cls._meta.app_label
    cache_key = u"{}-{}-{}".format(
        settings.PROJECT_SLUG, app_label, cls.__name__)
    for arg in extra_args:
        if not arg:
            cache_key += '-0'
        else:
            if type(arg) == dict:
                cache_key += '-' + "_".join([unicode(arg[k]) for k in arg])
            elif type(arg) in (list, tuple):
                cache_key += '-' + "_".join([unicode(v) for v in arg])
            else:
                cache_key += '-' + unicode(arg)
    cache_key = slugify(cache_key)
    if not cache_key.endswith('_current_keys') \
            and hasattr(cls, '_add_cache_key_to_refresh'):
        cls._add_cache_key_to_refresh(extra_args)
    if len(cache_key) >= 250:
        m = hashlib.md5()
        m.update(cache_key)
        cache_key = m.hexdigest()
    return cache_key, cache.get(cache_key)


def force_cached_label_changed(sender, **kwargs):
    if not kwargs.get('instance'):
        return
    kwargs['instance']._cached_label_checked = False
    cached_label_changed(sender, **kwargs)


def cached_label_changed(sender, **kwargs):
    if not kwargs.get('instance'):
        return
    instance = kwargs.get('instance')

    if hasattr(instance, 'test_obj'):
        instance.test_obj.reached(sender, **kwargs)

    if hasattr(instance, '_cached_label_checked') \
            and instance._cached_label_checked:
        return
    instance._cached_label_checked = True
    cached_labels = ['cached_label']
    if hasattr(sender, 'CACHED_LABELS'):
        cached_labels = sender.CACHED_LABELS
    changed = False
    for cached_label in cached_labels:
        lbl = getattr(instance, '_generate_' + cached_label)()
        if lbl != getattr(instance, cached_label):
            setattr(instance, cached_label, lbl)
            changed = True
    if changed:
        instance._search_updated = False
        if hasattr(instance, '_cascade_change') and instance._cascade_change:
            instance.skip_history_when_saving = True
        instance.save()
    updated = False
    if hasattr(instance, 'update_search_vector'):
        updated = instance.update_search_vector()
    if hasattr(instance, '_cached_labels_bulk_update'):
        updated = instance._cached_labels_bulk_update() or updated
    if not updated and hasattr(instance, '_get_associated_cached_labels'):
        for item in instance._get_associated_cached_labels():
            item._cascade_change = True
            if hasattr(instance, 'test_obj'):
                item.test_obj = instance.test_obj
            cached_label_changed(item.__class__, instance=item)


SHORTIFY_STR = ugettext(" (...)")


def shortify(lbl, number=20):
    if not lbl:
        lbl = ''
    if len(lbl) <= number:
        return lbl
    return lbl[:number - len(SHORTIFY_STR)] + SHORTIFY_STR


def mode(array):
    most = max(list(map(array.count, array)))
    return list(set(filter(lambda x: array.count(x) == most, array)))


def disable_for_loaddata(signal_handler):
    """
    Decorator that turns off signal handlers when loading fixture data.
    """

    @wraps(signal_handler)
    def wrapper(*args, **kwargs):
        if kwargs.get('raw'):
            return
        signal_handler(*args, **kwargs)
    return wrapper


def _get_image_link(item):
    # manage missing images
    if not item.thumbnail or not item.thumbnail.url or not item.image \
            or not item.image.url:
        return ""
    return mark_safe(u"""
    <div class="col col-lg-3">
    <div class="card">
        <div id="lightgallery-rand-img">
            <a href="{}">
                <img class="card-img-top" src="{}"/>
            </a>
        </div>
        <div class="card-body">
            <h4 class="card-title">{}</h4>
            <p class="card-text">{}</p>
            <a href="#" onclick="load_window(\'{}\')" class="btn btn-info">
                <i class="fa fa-info-circle" aria-hidden="true"></i> {}
            </a>
            <a href="." title="{}" class="btn btn-secondary">
                <i class="fa fa-random" aria-hidden="true"></i>
            </a><br/>
        </div>
    </div>
    </div>
    <script type="text/javascript">
        lightGallery(document.getElementById('lightgallery-rand-img'));
    </script>""".format(
        item.image.url,
        item.thumbnail.url,
        unicode(item.__class__._meta.verbose_name),
        unicode(item),
        reverse(item.SHOW_URL, args=[item.pk, '']),
        unicode(_(u"Information")),
        unicode(_(u"Load another random image?"))))


def get_random_item_image_link(request):
    from archaeological_operations.models import Operation
    from archaeological_context_records.models import ContextRecord
    from archaeological_finds.models import Find

    ope_image_nb, cr_image_nb, find_image_nb = 0, 0, 0
    q_ope = Operation.objects.filter(
        thumbnail__isnull=False, image__isnull=False
    ).exclude(thumbnail='').exclude(image='')
    q_cr = ContextRecord.objects.filter(
        thumbnail__isnull=False, image__isnull=False
    ).exclude(thumbnail='').exclude(image='')
    q_find = Find.objects.filter(
        thumbnail__isnull=False, image__isnull=False
    ).exclude(thumbnail='').exclude(image='')
    if request.user.has_perm('archaeological_operations.view_operation',
                             Operation):
        ope_image_nb = q_ope.count()
    if request.user.has_perm(
            'archaeological_context_records.view_contextrecord',
            ContextRecord):
        cr_image_nb = q_cr.count()
    if request.user.has_perm('archaeological_finds.view_find',
                             Find):
        find_image_nb = q_find.count()

    image_total = ope_image_nb + cr_image_nb + find_image_nb
    if not image_total:
        return ''

    image_nb = random.randint(0, image_total - 1)
    if image_nb >= 0 and image_nb < ope_image_nb:
        return _get_image_link(q_ope.all()[image_nb])
    if image_nb >= ope_image_nb and image_nb < (cr_image_nb + ope_image_nb):
        return _get_image_link(q_cr.all()[image_nb - ope_image_nb])
    if image_nb >= (cr_image_nb + ope_image_nb):
        return _get_image_link(q_find.all()[
            image_nb - ope_image_nb - cr_image_nb])
    # should never happen except in case of deletion during the excution
    return ''


def convert_coordinates_to_point(x, y, z=None, srid=4326):
    if z:
        geom = GEOSGeometry('POINT({} {} {})'.format(x, y, z), srid=srid)
    else:
        geom = GEOSGeometry('POINT({} {})'.format(x, y), srid=srid)
    if not geom.valid:
        raise forms.ValidationError(geom.valid_reason)
    return geom


def post_save_point(sender, **kwargs):
    """
    Convert raw x, y, z point to real geo field
    """
    if not kwargs.get('instance'):
        return
    instance = kwargs.get('instance')
    point = None
    point_2d = None
    if instance.x and instance.y and \
            instance.spatial_reference_system and \
            instance.spatial_reference_system.auth_name == 'EPSG' and \
            instance.spatial_reference_system.srid != 0:
        point_2d = convert_coordinates_to_point(
            instance.x, instance.y, srid=instance.spatial_reference_system.srid)
        if instance.z:
            point = convert_coordinates_to_point(
                instance.x, instance.y, instance.z,
                srid=instance.spatial_reference_system.srid)
    if point_2d != instance.point_2d or point != instance.point:
        instance.point = point
        instance.point_2d = point_2d
        instance.skip_history_when_saving = True
        instance.save()
    return


def create_slug(model, name, slug_attr='slug', max_length=100):
    base_slug = slugify(name)
    slug = base_slug[:max_length]
    final_slug = None
    idx = 1
    while not final_slug:
        if slug and not model.objects.filter(**{slug_attr:slug}).exists():
            final_slug = slug
            break
        slug = base_slug[:(max_length - 1 - len(str(idx)))] + "-" + str(idx)
        idx += 1
    return final_slug


def get_all_field_names(model):
    return list(set(chain.from_iterable(
        (field.name, field.attname) if hasattr(field, 'attname') else (
         field.name,)
        for field in model._meta.get_fields()
        if not (field.many_to_one and field.related_model is None)
    )))


def get_all_related_m2m_objects_with_model(model):
    return [
        (f, f.model if f.model != model else None)
        for f in model._meta.get_fields(include_hidden=True)
        if f.many_to_many and f.auto_created
    ]


def get_all_related_many_to_many_objects(model):
    return [
        f for f in model._meta.get_fields(include_hidden=True)
        if f.many_to_many and f.auto_created
    ]


def get_all_related_objects(model):
    return [
        f for f in model._meta.get_fields()
        if (f.one_to_many or f.one_to_one)
        and f.auto_created and not f.concrete
    ]


def merge_tsvectors(vectors):
    """
    Parse tsvector to merge them in one string
    :param vectors: list of tsvector string
    :return: merged tsvector
    """
    result_dict = {}
    for vector in vectors:
        if not vector:
            continue

        current_position = 0
        if result_dict:
            for key in result_dict:
                max_position = max(result_dict[key])
                if max_position > current_position:
                    current_position = max_position

        for dct_member in vector.split(" "):
            splitted = dct_member.split(':')
            key = ":".join(splitted[:-1])
            positions = splitted[-1]
            key = key[1:-1]  # remove quotes
            positions = [int(pos) + current_position
                         for pos in positions.split(',')]
            if key in result_dict:
                result_dict[key] += positions
            else:
                result_dict[key] = positions

    # {'lamelie': [1, 42, 5]} => {'lamelie': "1,42,5"}
    result_dict = {k: ",".join([str(val) for val in result_dict[k]])
                   for k in result_dict}
    # {'lamelie': "1,5", "hagarde": "2", "regarde": "4"} =>
    # "'lamelie':1,5 'hagarde':2 'regarde':4"
    result = " ".join(["'{}':{}".format(k, result_dict[k])
                       for k in result_dict])

    return result


def put_session_message(session_key, message, message_type):
    session = SessionStore(session_key=session_key)
    messages = []
    if 'messages' in session:
        messages = session['messages'][:]
    messages.append((unicode(message), message_type))
    session['messages'] = messages
    session.save()


def put_session_var(session_key, key, value):
    session = SessionStore(session_key=session_key)
    session[key] = value
    session.save()


def get_session_var(session_key, key):
    session = SessionStore(session_key=session_key)
    if key not in session:
        return
    return session[key]


def get_field_labels_from_path(model, path):
    """
    :param model: base model
    :param path: list of attribute starting from the base model
    :return: list of labels
    """
    labels = []
    for key in path:
        try:
            field = model._meta.get_field(key)
        except:
            labels.append(key)
            continue
        if hasattr(field, 'verbose_name'):
            labels.append(field.verbose_name)
        else:
            labels.append(key)
    return labels


def create_default_areas(models=None):
    # can be used on migrations if models are provided
    if not models:
        from ishtar_common.models import Area, Town, Department, State
    else:
        Area = models['area']
        Town = models['town']
        Department = models['department']
        State = models['state']

    areas = {}

    idx = 0
    for state in State.objects.all():
        slug = 'state-' + slugify(state.label)
        area, created = Area.objects.get_or_create(
            txt_idx=slug, defaults={'label': state.label})
        areas['state-{}'.format(state.pk)] = area
        if created:
            idx += 1
    print("\n* {} state areas added".format(idx))

    idx, idx2 = 0, 0
    for dep in Department.objects.all():
        slug = 'dep-' + slugify(dep.label)
        area, created = Area.objects.get_or_create(
            txt_idx=slug, defaults={'label': dep.label})
        areas['dep-' + dep.number] = area
        if created:
            idx += 1
        if not dep.state_id:
            continue
        state_slug = 'state-{}'.format(dep.state_id)
        if state_slug not in areas:
            continue
        if area.parent and area.parent.pk == areas[state_slug].pk:
            continue
        idx2 += 1
        area.parent = areas[state_slug]
        area.save()
    print(
        "* {} department areas added with {} associations to state".format(
            idx, idx2)
    )

    idx = 0
    for town in Town.objects.all():
        if not town.numero_insee or len(town.numero_insee) != 5:
            continue
        code_dep = 'dep-' + town.numero_insee[:2]
        code_dep_dom = 'dep-' + town.numero_insee[:3]
        if code_dep in areas:
            if not areas[code_dep].towns.filter(pk=town.pk).count():
                areas[code_dep].towns.add(town)
                idx += 1
        elif code_dep_dom in areas:
            if not areas[code_dep_dom].towns.filter(pk=town.pk).count():
                areas[code_dep_dom].towns.add(town)
                idx += 1

    print("* {} town associated to department area".format(idx))


def get_relations_for_graph(rel_model, obj_pk, above_relations=None,
                            equal_relations=None, treated=None, styles=None):
    """
    Get all above and equal relations of an object (get all child and parent
    relations)
    :param rel_model: the relation model concerned
    :param obj_pk: id of an object with relations
    :param above_relations: list of current above_relations
    :param equal_relations: list of current equal_relations
    :param treated: treated relation list to prevent circular call
    :param styles: current styles
    :return: above and equal relations list (each containing lists of two
    members)
    """
    if not above_relations:
        above_relations = []
    if not equal_relations:
        equal_relations = []
    if not treated:
        treated = []
    if not styles:
        styles = {}
    if obj_pk in treated:
        return above_relations, equal_relations, styles

    treated.append(obj_pk)
    q = rel_model.objects.filter(
        left_record_id=obj_pk,
        relation_type__logical_relation__isnull=False
    ).values('right_record_id', 'relation_type__logical_relation')
    if not q.count():
        return [], []
    for relation in q.all():
        logical_relation = relation['relation_type__logical_relation']
        right_record = relation['right_record_id']
        if not logical_relation:
            continue
        elif logical_relation == 'above'and \
                (obj_pk, right_record) not in above_relations:
            above_relations.append((obj_pk, right_record))
        elif logical_relation == 'bellow' and \
                (right_record, obj_pk) not in above_relations:
            above_relations.append((right_record, obj_pk))
        elif logical_relation == 'equal' and \
                (right_record, obj_pk) not in equal_relations and \
                (obj_pk, right_record) not in equal_relations:
            equal_relations.append((obj_pk, right_record))
        else:
            continue
        ar, er, substyles = get_relations_for_graph(
            rel_model, right_record, above_relations, equal_relations, treated,
            styles
        )
        styles.update(substyles)
        error_style = "color=red"
        for r in ar:
            if r not in above_relations:
                above_relations.append(r)
            reverse_rel = tuple(reversed(r))
            if reverse_rel in above_relations:
                # circular
                if r not in styles:
                    styles[r] = []
                if reverse_rel not in styles:
                    styles[reverse_rel] = []

                if error_style not in styles[r]:
                    styles[r].append(error_style)
                if error_style not in styles[reverse_rel]:
                    styles[reverse_rel].append(error_style)
            if r[0] == r[1]:
                # same entity
                if r not in styles:
                    styles[r] = []
                if error_style not in styles[r]:
                    styles[r].append("color=red")
        for r in er:
            if r not in equal_relations:
                equal_relations.append(r)
    return above_relations, equal_relations, styles


def generate_relation_graph(obj, debug=False):
    if not settings.DOT_BINARY:
        return

    model = obj.__class__
    rel_model = model._meta.get_field('right_relations').related_model

    # get relations
    above_relations, equal_relations, styles = get_relations_for_graph(
        rel_model, obj.pk)
    print(styles)
    if not above_relations and not equal_relations:
        obj.relation_image = None
        obj.save()
        return

    # generate dotfile
    dot_str = "digraph relations {\nnode [shape=box];\n"
    rel_str = ""
    described = []
    for list, directed in ((above_relations, True),
                           (equal_relations, False)):
        if directed:
            rel_str += "subgraph Dir {\n"
        else:
            rel_str += "subgraph NoDir {\nedge [dir=none,style=dashed];\n"
        for left_pk, right_pk in list:
            if left_pk not in described:
                described.append(left_pk)
                left = model.objects.get(pk=left_pk)
                style = 'label="{}"'.format(left.relation_label)
                if left.pk == obj.pk:
                    style += ',style=filled,fillcolor="#C6C0C0"'
                dot_str += u'item{}[{}];\n'.format(left.pk, style)
            if right_pk not in described:
                described.append(right_pk)
                right = model.objects.get(pk=right_pk)
                style = 'label="{}"'.format(right.relation_label)
                if right.pk == obj.pk:
                    style += ',style=filled,fillcolor="#C6C0C0"'
                dot_str += u'item{}[{}];\n'.format(right.pk, style)
            if not directed:  # on the same level
                rel_str += u"{{rank = same; item{}; item{};}}\n".format(
                    left_pk, right_pk)
            style = ""
            if (left_pk, right_pk) in styles:
                style = " [{}]".format(", ".join(styles[(left_pk, right_pk)]))
            rel_str += u'item{} -> item{}{};\n'.format(left_pk, right_pk, style)
        rel_str += "}\n"
    dot_str += rel_str + "\n}"

    tempdir = tempfile.mkdtemp("-ishtardot")
    dot_name = tempdir + os.path.sep + "relations.dot"
    with open(dot_name, 'w') as dot_file:
        dot_file.write(dot_str)

    # execute dot program
    args = (settings.DOT_BINARY, "-Tsvg", dot_name)

    svg_tmp_name = tempdir + os.path.sep + "relations.svg"

    with open(svg_tmp_name, "w") as svg_file:
        popen = subprocess.Popen(args, stdout=svg_file)
        popen.wait()

    with open(svg_tmp_name, "r") as svg_file:
        django_file = File(svg_file)
        obj.relation_image.save("relations.svg", django_file, save=True)

    if debug:
        print(u"DOT file: {}. Tmp SVG file: {}.".format(dot_name, svg_tmp_name))
        return
    shutil.rmtree(tempdir)