Django 1.11: adapt all get_field_* to match the current Options API

9 files changed, 80 insertions, 51 deletions

diff --git a/archaeological_context_records/models.py b/archaeological_context_records/models.py
index fe9f8a2bc..d71864302 100644
--- a/archaeological_context_records/models.py
+++ b/archaeological_context_records/models.py
@@ -384,8 +384,8 @@ class ContextRecord(BulkUpdatedItem, BaseHistorizedItem, ImageModel, OwnPerms,
                    join=settings.JOINT, where=where)
         with connection.cursor() as c:
             c.execute(sql, args)
-        cls._meta.get_field_by_name(
-            'base_finds')[0].related_model.cached_label_bulk_update(**kwargs)
+        cls._meta.get_field(
+            'base_finds').related_model.cached_label_bulk_update(**kwargs)
 
     @property
     def short_label(self):
diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py
index 0c71448b3..80c8656a7 100644
--- a/archaeological_operations/tests.py
+++ b/archaeological_operations/tests.py
@@ -50,11 +50,11 @@ OPERATION_TOWNS_FIXTURES = \
 
 FILE_FIXTURES = OPERATION_FIXTURES + [
     settings.ROOT_PATH +
-    '../archaeological_files/fixtures/initial_data.json']
+    '../archaeological_files/fixtures/initial_data-fr.json']
 
 FILE_TOWNS_FIXTURES = OPERATION_TOWNS_FIXTURES + [
     settings.ROOT_PATH +
-    '../archaeological_files/fixtures/initial_data.json']
+    '../archaeological_files/fixtures/initial_data-fr.json']
 
 
 class FileInit(object):
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index c89e46f0b..fa8c6a2e0 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -36,6 +36,9 @@ from django.db.models import Q
 from django.template.defaultfilters import slugify
 from django.utils.translation import ugettext_lazy as _
 
+from ishtar_common.utils import get_all_field_names
+
+
 NEW_LINE_BREAK = '#####@@@#####'
 
 RE_FILTER_CEDEX = re.compile("(.*) *(?: *CEDEX|cedex|Cedex|Cédex|cédex *\d*)")
@@ -487,7 +490,7 @@ class TypeFormater(StrChoiceFormater):
         values = copy.copy(self.defaults)
         values['label'] = value
         values['txt_idx'] = slugify(value)
-        if 'order' in self.model._meta.get_all_field_names():
+        if 'order' in get_all_field_names(self.model):
             order = 1
             q = self.model.objects.values('order').order_by('-order')
             if q.count():
@@ -1043,8 +1046,7 @@ class Importer(object):
         if self.test:
             return
         # manage unicity of items (mainly for updates)
-        if 'history_modifier' in \
-                self.OBJECT_CLS._meta.get_all_field_names():
+        if 'history_modifier' in get_all_field_names(self.OBJECT_CLS):
             data['history_modifier'] = self.history_modifier
 
         obj, created = self.get_object(self.OBJECT_CLS, data)
@@ -1327,7 +1329,7 @@ class Importer(object):
             if not notempty:
                 continue
 
-            field_names = model._meta.get_all_field_names()
+            field_names = get_all_field_names(model)
             for v in vals:
                 if 'history_modifier' in field_names:
                     if 'defaults' not in v:
@@ -1444,13 +1446,12 @@ class Importer(object):
             self._set_importer_trigger(cls, attribute, data)
             return
         try:
-            field_object, model, direct, m2m = \
-                cls._meta.get_field_by_name(attribute)
+            field_object = cls._meta.get_field(attribute)
         except FieldDoesNotExist:
             raise ImporterError(unicode(
                 _(u"Importer configuration error: field \"{}\" does not exist "
                   u"for {}.")).format(attribute, cls._meta.verbose_name))
-        if m2m:
+        if field_object.many_to_many:
             m2ms += self._get_field_m2m(attribute, data, c_path,
                                         new_created, field_object)
             return
@@ -1470,8 +1471,7 @@ class Importer(object):
             # we treat only dict formated values
             return
         # put history_modifier for every created item
-        if 'history_modifier' in \
-           field_object.rel.to._meta.get_all_field_names():
+        if 'history_modifier' in get_all_field_names(field_object.rel.to):
             data[attribute]['history_modifier'] = \
                 self.history_modifier
         try:
@@ -1506,9 +1506,8 @@ class Importer(object):
                     data.pop(attribute)
                     continue
                 if not data[attribute]:
-                    field_object, model, direct, m2m = \
-                        cls._meta.get_field_by_name(attribute)
-                    if m2m:
+                    field_object = cls._meta.get_field(attribute)
+                    if field_object.many_to_many:
                         data.pop(attribute)
                     continue
                 if attribute != '__force_new':
diff --git a/ishtar_common/model_merging.py b/ishtar_common/model_merging.py
index be2867cb6..a390f233c 100644
--- a/ishtar_common/model_merging.py
+++ b/ishtar_common/model_merging.py
@@ -6,6 +6,9 @@ from django.db.models import Model
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.core.exceptions import ObjectDoesNotExist
 
+from ishtar_common.utils import get_all_related_many_to_many_objects, \
+    get_all_related_objects
+
 
 def get_models():
     _apps = apps.app_configs.items()
@@ -69,7 +72,7 @@ def merge_model_objects(primary_object, alias_objects=[], keep_old=False):
     for alias_object in alias_objects:
         # Migrate all foreign key references from alias object to primary
         # object.
-        for related_object in alias_object._meta.get_all_related_objects():
+        for related_object in get_all_related_objects(alias_object):
             # The variable name on the alias_object model.
             alias_varname = related_object.get_accessor_name()
             # The variable name on the related model.
@@ -85,7 +88,7 @@ def merge_model_objects(primary_object, alias_objects=[], keep_old=False):
         # Migrate all many to many references from alias object to primary
         # object.
         related_many_objects = \
-            alias_object._meta.get_all_related_many_to_many_objects()
+            get_all_related_many_to_many_objects(alias_object)
         related_many_object_names = set()
         for related_many_object in related_many_objects:
             alias_varname = related_many_object.get_accessor_name()
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index 082991824..19f557511 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -22,10 +22,8 @@ Models description
 """
 from cStringIO import StringIO
 import copy
-import csv
 import datetime
 from PIL import Image
-from importlib import import_module
 import logging
 import os
 from os.path import isfile, join
@@ -35,24 +33,18 @@ import shutil
 from subprocess import Popen, PIPE
 import tempfile
 import time
-import unicodecsv
-import zipfile
 
 from django.conf import settings
 from django.core.cache import cache
-from django.core.exceptions import ObjectDoesNotExist, ValidationError, \
-    SuspiciousOperation
-from django.core.files import File
+from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.core.validators import validate_slug
 from django.core.urlresolvers import reverse, NoReverseMatch
 from django.db.utils import DatabaseError
 from django.db.models import Q, Max, Count
-from django.db.models.base import ModelBase
-from django.db.models.signals import post_save, pre_delete, post_delete
+from django.db.models.signals import post_save, post_delete
 from django.utils.functional import lazy
-from django.utils.translation import ugettext_lazy as _, ugettext, \
-    pgettext_lazy
+from django.utils.translation import ugettext_lazy as _
 
 from django.utils.safestring import SafeUnicode, mark_safe
 from django.template.defaultfilters import slugify
@@ -65,10 +57,8 @@ from django.contrib.gis.db import models
 from simple_history.models import HistoricalRecords as BaseHistoricalRecords
 
 from ishtar_common.model_merging import merge_model_objects
-from ishtar_common.utils import get_cache, disable_for_loaddata, create_slug
-from ishtar_common.data_importer import Importer, ImportFormater, \
-    IntegerFormater, FloatFormater, UnicodeFormater, DateFormater, \
-    TypeFormater, YearFormater, StrToBoolean, FileFormater
+from ishtar_common.utils import get_cache, disable_for_loaddata, create_slug,\
+    get_all_field_names
 
 from ishtar_common.models_imports import ImporterModel, ImporterType, \
     ImporterDefault, ImporterDefaultValues, ImporterColumn, \
@@ -143,7 +133,7 @@ class ValueGetter(object):
         if not prefix:
             prefix = self._prefix
         values = {}
-        for field_name in self._meta.get_all_field_names():
+        for field_name in get_all_field_names(self):
             if not hasattr(self, field_name):
                 continue
             value = getattr(self, field_name)
@@ -180,7 +170,7 @@ class ValueGetter(object):
         if not prefix:
             prefix = cls._prefix
         values = {}
-        for field_name in cls._meta.get_all_field_names():
+        for field_name in get_all_field_names(cls):
             values[prefix + field_name] = ''
         return values
 
@@ -789,12 +779,12 @@ class ImageModel(models.Model):
 
     def __init__(self, *args, **kwargs):
         super(ImageModel, self).__init__(*args, **kwargs)
-        image = self._meta.get_field_by_name("image")[0]
+        image = self._meta.get_field("image")
         IMAGE_PREFIX = self.IMAGE_PREFIX
         if not IMAGE_PREFIX.endswith('/'):
             IMAGE_PREFIX += u'/'
         image.upload_to = IMAGE_PREFIX
-        thumbnail = self._meta.get_field_by_name("thumbnail")[0]
+        thumbnail = self._meta.get_field("thumbnail")
         thumbnail.upload_to = IMAGE_PREFIX + "thumbs/"
 
     def has_changed(self, field):
@@ -939,8 +929,8 @@ class BaseHistorizedItem(Imported):
             item._next = None
         item.history_date = historized[step].history_date
         model = self.__class__
-        for k in model._meta.get_all_field_names():
-            field = model._meta.get_field_by_name(k)[0]
+        for k in get_all_field_names(model):
+            field = model._meta.get_field(k)
             if hasattr(field, 'rel') and field.rel:
                 if not hasattr(item, k + '_id'):
                     setattr(item, k, getattr(self, k))
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index dcb02c27e..26b034522 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -37,7 +37,8 @@ from django.db.models.signals import pre_delete
 from django.template.defaultfilters import slugify
 from django.utils.translation import ugettext_lazy as _, pgettext_lazy
 
-from ishtar_common.utils import create_slug
+from ishtar_common.utils import create_slug, \
+    get_all_related_m2m_objects_with_model
 from ishtar_common.data_importer import Importer, ImportFormater, \
     IntegerFormater, FloatFormater, UnicodeFormater, DateFormater, \
     TypeFormater, YearFormater, StrToBoolean, FileFormater
@@ -889,8 +890,7 @@ class Import(models.Model):
 
     def get_all_imported(self):
         imported = []
-        for related, zorg in \
-                self._meta.get_all_related_m2m_objects_with_model():
+        for related, zorg in get_all_related_m2m_objects_with_model(self):
             accessor = related.get_accessor_name()
             imported += [(accessor, obj)
                          for obj in getattr(self, accessor).all()]
diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py
index 9eda2d22f..c6a4032f0 100644
--- a/ishtar_common/utils.py
+++ b/ishtar_common/utils.py
@@ -17,10 +17,11 @@
 
 # See the file COPYING for details.
 
+import datetime
 from functools import wraps
+from itertools import chain
 import hashlib
 import random
-import datetime
 
 from django import forms
 from django.conf import settings
@@ -255,3 +256,36 @@ def create_slug(model, name, slug_attr='slug', max_length=100):
         slug = base_slug[:(max_length - 1 - len(str(idx)))] + "-" + str(idx)
         idx += 1
     return final_slug
+
+
+def get_all_field_names(model):
+    return list(set(chain.from_iterable(
+        (field.name, field.attname) if hasattr(field, 'attname') else (
+         field.name,)
+        for field in model._meta.get_fields()
+        if not (field.many_to_one and field.related_model is None)
+    )))
+
+
+def get_all_related_m2m_objects_with_model(model):
+    return [
+        (f, f.model if f.model != model else None)
+        for f in model._meta.get_fields(include_hidden=True)
+        if f.many_to_many and f.auto_created
+    ]
+
+
+def get_all_related_many_to_many_objects(model):
+    return [
+        f for f in model._meta.get_fields(include_hidden=True)
+        if f.many_to_many and f.auto_created
+    ]
+
+
+def get_all_related_objects(model):
+    return [
+        f for f in model._meta.get_fields()
+        if (f.one_to_many or f.one_to_one)
+        and f.auto_created and not f.concrete
+    ]
+
diff --git a/ishtar_common/views.py b/ishtar_common/views.py
index 0f1397758..8ab07fc00 100644
--- a/ishtar_common/views.py
+++ b/ishtar_common/views.py
@@ -72,7 +72,8 @@ from archaeological_finds.forms import DashboardTreatmentForm, \
 
 from ishtar_common.forms import FinalForm, FinalDeleteForm
 from ishtar_common.widgets import JQueryAutoComplete
-from ishtar_common.utils import get_random_item_image_link, shortify
+from ishtar_common.utils import get_random_item_image_link, shortify, \
+    get_all_field_names
 from ishtar_common import forms_common as forms
 from ishtar_common import wizards
 from ishtar_common.models import HistoryError, PRIVATE_FIELDS, \
@@ -677,8 +678,8 @@ def get_item(model, func_name, default_name, extra_request_keys=[],
         else:
             my_relation_types_prefix = copy(relation_types_prefix)
 
-        fields = [model._meta.get_field_by_name(k)[0]
-                  for k in model._meta.get_all_field_names()]
+        fields = [model._meta.get_field(k)
+                  for k in get_all_field_names(model)]
         request_keys = dict([
             (field.name,
              field.name + (hasattr(field, 'rel') and field.rel and '__pk'
@@ -690,8 +691,8 @@ def get_item(model, func_name, default_name, extra_request_keys=[],
                     continue
                 associated_model = globals()[associated_model]
             associated_fields = [
-                associated_model._meta.get_field_by_name(k)[0]
-                for k in associated_model._meta.get_all_field_names()]
+                associated_model._meta.get_field(k)
+                for k in get_all_field_names(associated_model)]
             request_keys.update(
                 dict([(key + "__" + field.name,
                        key + "__" + field.name +
diff --git a/ishtar_common/wizards.py b/ishtar_common/wizards.py
index a82b68d5b..5217eafae 100644
--- a/ishtar_common/wizards.py
+++ b/ishtar_common/wizards.py
@@ -40,7 +40,9 @@ from django.shortcuts import render_to_response, redirect
 from django.template import Context, RequestContext, loader
 from django.utils.datastructures import MultiValueDict as BaseMultiValueDict
 from django.utils.translation import ugettext_lazy as _
-import models
+
+from ishtar_common import models
+from ishtar_common.utils import get_all_field_names
 
 logger = logging.getLogger(__name__)
 
@@ -528,7 +530,7 @@ class Wizard(NamedUrlWizardView):
             for k in dct:
                 if k.startswith('pk'):
                     continue
-                if k not in obj.__class__._meta.get_all_field_names():
+                if k not in get_all_field_names(obj.__class__):
                     continue
                 # False set to None for images and files
                 if not k.endswith('_id') and (
@@ -608,7 +610,7 @@ class Wizard(NamedUrlWizardView):
             if 'pk' in dct:
                 dct.pop('pk')
             # remove non relevant fields
-            all_field_names = self.get_saved_model()._meta.get_all_field_names()
+            all_field_names = get_all_field_names(self.get_saved_model())
             for k in dct.copy():
                 if not (k.endswith('_id') and k[:-3] in all_field_names) \
                         and k not in all_field_names and \