Format - black: ishtar_common

1 files changed, 586 insertions, 433 deletions

diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index 108e52d7b..b777850c9 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -39,7 +39,7 @@ from django.utils.translation import ugettext_lazy as _
 from ishtar_common.utils import debug_line_no, get_all_field_names, update_data
 
 
-NEW_LINE_BREAK = '#####@@@#####'
+NEW_LINE_BREAK = "#####@@@#####"
 
 RE_FILTER_CEDEX = re.compile("(.*) *(?: *CEDEX|cedex|Cedex|Cédex|cédex *\d*)")
 
@@ -47,25 +47,42 @@ RE_FILTER_CEDEX = re.compile("(.*) *(?: *CEDEX|cedex|Cedex|Cédex|cédex *\d*)")
 def post_importer_action(func):
     def wrapper(self, context, value):
         return func(self, context, value)
-    wrapper.importer_trigger = 'post'
+
+    wrapper.importer_trigger = "post"
     return wrapper
 
 
 def pre_importer_action(func):
     def wrapper(self, context, value):
         return func(self, context, value)
-    wrapper.importer_trigger = 'pre'
+
+    wrapper.importer_trigger = "pre"
     return wrapper
 
 
 class ImportFormater(object):
-    def __init__(self, field_name, formater=None, required=True, through=None,
-                 through_key=None, through_dict=None,
-                 through_unicity_keys=None, duplicate_fields=None, regexp=None,
-                 regexp_formater_args=None, force_value=None,
-                 post_processing=False, concat=False, concat_str=False,
-                 comment="", force_new=None, export_field_name=None,
-                 value_format=None, label=""):
+    def __init__(
+        self,
+        field_name,
+        formater=None,
+        required=True,
+        through=None,
+        through_key=None,
+        through_dict=None,
+        through_unicity_keys=None,
+        duplicate_fields=None,
+        regexp=None,
+        regexp_formater_args=None,
+        force_value=None,
+        post_processing=False,
+        concat=False,
+        concat_str=False,
+        comment="",
+        force_new=None,
+        export_field_name=None,
+        value_format=None,
+        label="",
+    ):
         self.field_name = field_name
         if export_field_name:
             self.export_field_name = export_field_name
@@ -117,28 +134,33 @@ class ImportFormater(object):
     def report_error(self, *args):
         return
 
-    def init(self, vals, output=None, choose_default=False,
-             import_instance=None, user=None):
+    def init(
+        self, vals, output=None, choose_default=False, import_instance=None, user=None
+    ):
         try:
             lst = iter(self.formater)
         except TypeError:
             lst = [self.formater]
         for formater in lst:
             if formater:
-                formater.check(vals, output, self.comment,
-                               choose_default=choose_default,
-                               import_instance=import_instance,
-                               user=user)
+                formater.check(
+                    vals,
+                    output,
+                    self.comment,
+                    choose_default=choose_default,
+                    import_instance=import_instance,
+                    user=user,
+                )
 
     def post_process(self, obj, context, value, owner=None):
         raise NotImplemented()
 
 
 class ImporterError(Exception):
-    STANDARD = 'S'
-    HEADER = 'H'
+    STANDARD = "S"
+    HEADER = "H"
 
-    def __init__(self, message, type='S'):
+    def __init__(self, message, type="S"):
         self.msg = message
         self.type = type
 
@@ -148,13 +170,20 @@ class ImporterError(Exception):
 
 class Formater(object):
     def __init__(self, *args, **kwargs):
-        self.db_target = kwargs.get('db_target', None)
+        self.db_target = kwargs.get("db_target", None)
 
     def format(self, value):
         return value
 
-    def check(self, values, output=None, comment='', choose_default=False,
-              import_instance=None, user=None):
+    def check(
+        self,
+        values,
+        output=None,
+        comment="",
+        choose_default=False,
+        import_instance=None,
+        user=None,
+    ):
         return
 
     def init_db_target(self, user=None):
@@ -163,17 +192,16 @@ class Formater(object):
     def _base_target_filter(self, user=None):
         # set for all users
         q_or = (
-            Q(associated_import__isnull=True) &
-            Q(associated_user__isnull=True) &
-            Q(associated_group__isnull=True)
+            Q(associated_import__isnull=True)
+            & Q(associated_user__isnull=True)
+            & Q(associated_group__isnull=True)
         )
-        if hasattr(self, 'import_instance') and self.import_instance:
+        if hasattr(self, "import_instance") and self.import_instance:
             # set for current import
             q_or = q_or | Q(associated_import=self.import_instance)
             if self.import_instance.associated_group:
                 # set for associated group
-                q_or = q_or | Q(
-                    associated_group=self.import_instance.associated_group)
+                q_or = q_or | Q(associated_group=self.import_instance.associated_group)
         if user:
             # set for current user
             q_or = q_or | Q(associated_user=user)
@@ -192,17 +220,25 @@ class ChoiceChecker(object):
     def report_new(self, comment):
         if not self.new_keys:
             return
-        msg = "For \"%s\" these new associations have been made:\n" % comment
-        sys.stderr.write(msg.encode('utf-8'))
+        msg = 'For "%s" these new associations have been made:\n' % comment
+        sys.stderr.write(msg.encode("utf-8"))
         for k in self.new_keys:
             msg = '"%s";"%s"\n' % (k, self.new_keys[k])
-            sys.stderr.write(msg.encode('utf-8'))
+            sys.stderr.write(msg.encode("utf-8"))
 
 
 class UnicodeFormater(Formater):
-    def __init__(self, max_length=None, clean=False, re_filter=None,
-                 notnull=False, prefix='', db_target=None,
-                 import_instance=None, many_split=None):
+    def __init__(
+        self,
+        max_length=None,
+        clean=False,
+        re_filter=None,
+        notnull=False,
+        prefix="",
+        db_target=None,
+        import_instance=None,
+        many_split=None,
+    ):
         self.max_length = max_length
         self.db_target = db_target
         self.clean = clean
@@ -217,7 +253,7 @@ class UnicodeFormater(Formater):
             if type(value) != str:
                 value = str(value.strip())
             vals = []
-            for v in value.split('\n'):
+            for v in value.split("\n"):
                 v = v.strip()
                 if v:
                     vals.append(v)
@@ -236,9 +272,12 @@ class UnicodeFormater(Formater):
             return
         if self.max_length and len(value) > self.max_length:
             raise ValueError(
-                _("\"%(value)s\" is too long. The max length is %(length)d "
-                  "characters.") % {'value': value,
-                                    'length': self.max_length})
+                _(
+                    '"%(value)s" is too long. The max length is %(length)d '
+                    "characters."
+                )
+                % {"value": value, "length": self.max_length}
+            )
         if self.notnull and not value:
             return
         if value:
@@ -249,25 +288,23 @@ class UnicodeFormater(Formater):
 class BooleanFormater(Formater):
     def format(self, value):
         value = value.strip().upper()
-        if value in ('1', 'OUI', 'VRAI', 'YES', 'TRUE'):
+        if value in ("1", "OUI", "VRAI", "YES", "TRUE"):
             return True
-        if value in ('', '0', 'NON', 'FAUX', 'NO', 'FALSE'):
+        if value in ("", "0", "NON", "FAUX", "NO", "FALSE"):
             return False
-        raise ValueError(_("\"%(value)s\" not equal to yes or no") % {
-            'value': value})
+        raise ValueError(_('"%(value)s" not equal to yes or no') % {"value": value})
 
 
 class FloatFormater(Formater):
     def format(self, value):
-        value = value.strip().replace(',', '.')
-        value = value.replace(' ', '')
+        value = value.strip().replace(",", ".")
+        value = value.replace(" ", "")
         if not value:
             return
         try:
             return float(value)
         except ValueError:
-            raise ValueError(_("\"%(value)s\" is not a float") % {
-                'value': value})
+            raise ValueError(_('"%(value)s" is not a float') % {"value": value})
 
 
 class InseeFormater(Formater):
@@ -276,11 +313,12 @@ class InseeFormater(Formater):
     The syntax "CodeINSEE-Year" is accepted (Ishtar trick) in order to manage
     old INSEE (year is the date of creation)
     """
-    ERROR = _("\"{value}\" is not an appropriate INSEE code")
+
+    ERROR = _('"{value}" is not an appropriate INSEE code')
 
     def format(self, value):
         value = value.strip()
-        exp = value.split('-')
+        exp = value.split("-")
         code = exp[0]
         try:
             int(code)
@@ -293,7 +331,7 @@ class InseeFormater(Formater):
         elif len(exp) == 1:
             return code
         try:
-            datetime.datetime.strptime(exp[1], '%Y')
+            datetime.datetime.strptime(exp[1], "%Y")
         except ValueError:
             raise ValueError(str(self.ERROR).format(value))
         return code + "-" + exp[1]
@@ -308,8 +346,7 @@ class YearFormater(Formater):
             value = int(value)
             assert value > 0 and value < (datetime.date.today().year + 30)
         except (ValueError, AssertionError):
-            raise ValueError(_("\"%(value)s\" is not a valid date") % {
-                'value': value})
+            raise ValueError(_('"%(value)s" is not a valid date') % {"value": value})
         return value
 
 
@@ -322,28 +359,34 @@ class YearNoFuturFormater(Formater):
             value = int(value)
             assert value > 0 and value < datetime.date.today().year
         except (ValueError, AssertionError):
-            raise ValueError(_("\"%(value)s\" is not a valid date") % {
-                'value': value})
+            raise ValueError(_('"%(value)s" is not a valid date') % {"value": value})
         return value
 
 
 class IntegerFormater(Formater):
     def format(self, value):
         value = value.strip()
-        value = value.replace(' ', '')
+        value = value.replace(" ", "")
         if not value:
             return
         try:
             return int(value)
         except ValueError:
-            raise ValueError(_("\"%(value)s\" is not an integer") % {
-                'value': value})
+            raise ValueError(_('"%(value)s" is not an integer') % {"value": value})
 
 
 class StrChoiceFormater(Formater, ChoiceChecker):
-    def __init__(self, choices, strict=False, equiv_dict=None, model=None,
-                 cli=False, many_split='', db_target=None,
-                 import_instance=None):
+    def __init__(
+        self,
+        choices,
+        strict=False,
+        equiv_dict=None,
+        model=None,
+        cli=False,
+        many_split="",
+        db_target=None,
+        import_instance=None,
+    ):
         if not equiv_dict:
             equiv_dict = {}
         self.choices = list(choices)
@@ -395,23 +438,30 @@ class StrChoiceFormater(Formater, ChoiceChecker):
     def prepare(self, value):
         return str(value).strip()
 
-    def _get_choices(self, comment=''):
+    def _get_choices(self, comment=""):
         msgstr = comment + " - "
-        msgstr += str(_("Choice for \"%s\" is not available. "
-                        "Which one is relevant?\n"))
+        msgstr += str(
+            _('Choice for "%s" is not available. ' "Which one is relevant?\n")
+        )
         idx = -1
         for idx, choice in enumerate(self.choices):
             msgstr += "%d. %s\n" % (idx + 1, choice[1])
         idx += 2
         if self.create:
-            msgstr += str(_("%d. None of the above - create new")) % idx \
-                + "\n"
+            msgstr += str(_("%d. None of the above - create new")) % idx + "\n"
             idx += 1
         msgstr += str(_("%d. None of the above - skip")) % idx + "\n"
         return msgstr, idx
 
-    def check(self, values, output=None, comment='', choose_default=False,
-              import_instance=None, user=None):
+    def check(
+        self,
+        values,
+        output=None,
+        comment="",
+        choose_default=False,
+        import_instance=None,
+        user=None,
+    ):
         self.init_db_target(user)
 
         """
@@ -433,7 +483,7 @@ class StrChoiceFormater(Formater, ChoiceChecker):
                     except IntegrityError:
                         pass
         """
-        if (not output or output == 'silent') and not choose_default:
+        if (not output or output == "silent") and not choose_default:
             return
         if self.many_split:
             new_values = []
@@ -446,7 +496,7 @@ class StrChoiceFormater(Formater, ChoiceChecker):
             value = self.prepare(value)
             if value in self.equiv_dict:
                 continue
-            if output != 'cli' and not choose_default:
+            if output != "cli" and not choose_default:
                 self.missings.add(value)
                 continue
             msgstr, idx = self._get_choices(comment)
@@ -455,7 +505,7 @@ class StrChoiceFormater(Formater, ChoiceChecker):
                 res = 1
             while res not in range(1, idx + 1):
                 msg = msgstr % value
-                sys.stdout.write(msg.encode('utf-8'))
+                sys.stdout.write(msg.encode("utf-8"))
                 sys.stdout.write("\n>>> ")
                 res = input()
                 try:
@@ -472,14 +522,16 @@ class StrChoiceFormater(Formater, ChoiceChecker):
                 self.new_keys[value] = v
             elif self.create and res == len(self.choices):
                 self.equiv_dict[value] = self.new(base_value)
-                self.choices.append((self.equiv_dict[value].pk,
-                                     str(self.equiv_dict[value])))
+                self.choices.append(
+                    (self.equiv_dict[value].pk, str(self.equiv_dict[value]))
+                )
                 self.new_keys[value] = str(self.equiv_dict[value])
             else:
                 self.equiv_dict[value] = None
             if self.equiv_dict[value] and self.db_target:
                 from ishtar_common.models import TargetKey
-                q = {'target': self.db_target, 'key': value}
+
+                q = {"target": self.db_target, "key": value}
                 query = TargetKey.objects.filter(**q)
                 query = query.filter(self._base_target_filter(user))
                 if query.count():
@@ -488,29 +540,30 @@ class StrChoiceFormater(Formater, ChoiceChecker):
                     target.is_set = True
                     target.save()
                 else:
-                    q['associated_import'] = import_instance
+                    q["associated_import"] = import_instance
                     with transaction.atomic():
-                        q['value'] = self.equiv_dict[value]
-                        q['is_set'] = True
+                        q["value"] = self.equiv_dict[value]
+                        q["is_set"] = True
                         try:
                             TargetKey.objects.create(**q)
                         except IntegrityError:
                             pass
-        if output == 'db' and self.db_target:
+        if output == "db" and self.db_target:
             from ishtar_common.models import TargetKey
+
             for missing in self.missings:
-                q = {'target': self.db_target, 'key': missing}
+                q = {"target": self.db_target, "key": missing}
                 query = TargetKey.objects.filter(**q)
                 query = query.filter(self._base_target_filter(user))
                 if query.count():
                     continue
                 with transaction.atomic():
-                    q['associated_import'] = import_instance
+                    q["associated_import"] = import_instance
                     try:
                         TargetKey.objects.create(**q)
                     except IntegrityError:
                         pass
-        if output == 'cli':
+        if output == "cli":
             self.report_new(comment)
 
     def new(self, value):
@@ -525,13 +578,20 @@ class StrChoiceFormater(Formater, ChoiceChecker):
         if not self.strict:
             value = slugify(value)
         if value in self.equiv_dict:
-            self.match_table[origin_value] = self.equiv_dict[value] or ''
+            self.match_table[origin_value] = self.equiv_dict[value] or ""
             return self.equiv_dict[value]
 
 
 class TypeFormater(StrChoiceFormater):
-    def __init__(self, model, cli=False, defaults=None, many_split=False,
-                 db_target=None, import_instance=None):
+    def __init__(
+        self,
+        model,
+        cli=False,
+        defaults=None,
+        many_split=False,
+        db_target=None,
+        import_instance=None,
+    ):
         if not defaults:
             defaults = {}
         self.create = True
@@ -559,20 +619,19 @@ class TypeFormater(StrChoiceFormater):
 
     def new(self, value):
         values = copy.copy(self.defaults)
-        values['label'] = value
-        values['txt_idx'] = slugify(value)
-        if 'order' in get_all_field_names(self.model):
+        values["label"] = value
+        values["txt_idx"] = slugify(value)
+        if "order" in get_all_field_names(self.model):
             order = 1
-            q = self.model.objects.values('order').order_by('-order')
+            q = self.model.objects.values("order").order_by("-order")
             if q.count():
-                order = q.all()[0]['order'] or 1
-            values['order'] = order
+                order = q.all()[0]["order"] or 1
+            values["order"] = order
         return self.model.objects.create(**values)
 
 
 class DateFormater(Formater):
-    def __init__(self, date_formats=None, db_target=None,
-                 import_instance=None):
+    def __init__(self, date_formats=None, db_target=None, import_instance=None):
         if not date_formats:
             date_formats = ["%d/%m/%Y"]
         self.date_formats = date_formats
@@ -590,8 +649,7 @@ class DateFormater(Formater):
                 return datetime.datetime.strptime(value, date_format).date()
             except:
                 continue
-        raise ValueError(_("\"%(value)s\" is not a valid date") % {
-            'value': value})
+        raise ValueError(_('"%(value)s" is not a valid date') % {"value": value})
 
 
 class FileFormater(Formater):
@@ -602,31 +660,38 @@ class FileFormater(Formater):
         if not value:
             return
         zp = zipfile.ZipFile(archive)
-        value = value.strip().replace('\\', '/')
-        items = value.replace('/', '_').split('.')
-        base_dir = settings.MEDIA_ROOT + 'imported'
+        value = value.strip().replace("\\", "/")
+        items = value.replace("/", "_").split(".")
+        base_dir = settings.MEDIA_ROOT + "imported"
         if not os.path.isdir(base_dir):
             os.mkdir(base_dir)
-        filename = base_dir + os.sep + \
-            ".".join(items[:-1]) + '.' + items[-1]
+        filename = base_dir + os.sep + ".".join(items[:-1]) + "." + items[-1]
 
         try:
-            with open(filename, 'wb') as f:
+            with open(filename, "wb") as f:
                 with zp.open(value) as z:
                     f.write(z.read())
-            f = open(filename, 'rb')
+            f = open(filename, "rb")
             my_file = File(f)
             # manually set the file size because of an issue with TempFile
             my_file.size = os.stat(filename).st_size
             return my_file
         except KeyError:
-            raise ValueError(_("\"%(value)s\" is not a valid path for the "
-                               "given archive") % {'value': value})
+            raise ValueError(
+                _('"%(value)s" is not a valid path for the ' "given archive")
+                % {"value": value}
+            )
 
 
 class StrToBoolean(Formater, ChoiceChecker):
-    def __init__(self, choices=None, cli=False, strict=False, db_target=None,
-                 import_instance=None):
+    def __init__(
+        self,
+        choices=None,
+        cli=False,
+        strict=False,
+        db_target=None,
+        import_instance=None,
+    ):
         if not choices:
             choices = {}
         self.dct = copy.copy(choices)
@@ -656,14 +721,21 @@ class StrToBoolean(Formater, ChoiceChecker):
             value = slugify(value)
         return value
 
-    def check(self, values, output=None, comment='', choose_default=False,
-              import_instance=None, user=None):
-        if (not output or output == 'silent') and not choose_default:
+    def check(
+        self,
+        values,
+        output=None,
+        comment="",
+        choose_default=False,
+        import_instance=None,
+        user=None,
+    ):
+        if (not output or output == "silent") and not choose_default:
             return
         msgstr = comment + " - "
-        msgstr += str(_(
-            "Choice for \"%s\" is not available. "
-            "Which one is relevant?\n"))
+        msgstr += str(
+            _('Choice for "%s" is not available. ' "Which one is relevant?\n")
+        )
         msgstr += "1. True\n"
         msgstr += "2. False\n"
         msgstr += "3. Empty\n"
@@ -671,7 +743,7 @@ class StrToBoolean(Formater, ChoiceChecker):
             value = self.prepare(value)
             if value in self.dct:
                 continue
-            if output != 'cli' and not choose_default:
+            if output != "cli" and not choose_default:
                 self.missings.add(value)
                 continue
             res = None
@@ -679,7 +751,7 @@ class StrToBoolean(Formater, ChoiceChecker):
                 res = 1
             while res not in range(1, 4):
                 msg = msgstr % value
-                sys.stdout.write(msg.encode('utf-8'))
+                sys.stdout.write(msg.encode("utf-8"))
                 sys.stdout.write("\n>>> ")
                 res = input()
                 try:
@@ -693,17 +765,21 @@ class StrToBoolean(Formater, ChoiceChecker):
             else:
                 self.dct[value] = None
             self.new_keys[value] = str(self.dct[value])
-        if output == 'db' and self.db_target:
+        if output == "db" and self.db_target:
             from ishtar_common.models import TargetKey
+
             for missing in self.missings:
                 try:
-                    q = {'target': self.db_target, 'key': missing,
-                         'associated_import': import_instance}
+                    q = {
+                        "target": self.db_target,
+                        "key": missing,
+                        "associated_import": import_instance,
+                    }
                     if not TargetKey.objects.filter(**q).count():
                         TargetKey.objects.create(**q)
                 except IntegrityError:
                     pass
-        if output == 'cli':
+        if output == "cli":
             self.report_new(comment)
 
     def format(self, value):
@@ -714,11 +790,12 @@ class StrToBoolean(Formater, ChoiceChecker):
             self.match_table[origin_value] = _(val)
             return self.dct[value]
 
+
 logger = logging.getLogger(__name__)
 
 
 def get_object_from_path(obj, path):
-    for k in path.split('__')[:-1]:
+    for k in path.split("__")[:-1]:
         if not hasattr(obj, k):
             return
         obj = getattr(obj, k)
@@ -726,8 +803,8 @@ def get_object_from_path(obj, path):
 
 
 class Importer(object):
-    SLUG = ''
-    NAME = ''
+    SLUG = ""
+    NAME = ""
     DESC = ""
     LINE_FORMAT = []
     OBJECT_CLS = None
@@ -737,22 +814,27 @@ class Importer(object):
     EXTRA_DEFAULTS = {}
     DEFAULTS = {}
     ERRORS = {
-        'header_check': _(
+        "header_check": _(
             "The given file is not correct. Check the file "
             "format. If you use a CSV file: check that column separator "
             "and encoding are similar to the ones used by the reference "
-            "file."),
-        'too_many_cols': _("Too many cols (%(user_col)d) when "
-                           "maximum is %(ref_col)d"),
-        'no_data': _("No data provided"),
-        'value_required': _("Value is required"),
-        'not_enough_cols': _("At least %d columns must be filled"),
-        'regex_not_match': _("The regexp doesn't match."),
-        'improperly_configured': _(
+            "file."
+        ),
+        "too_many_cols": _(
+            "Too many cols (%(user_col)d) when " "maximum is %(ref_col)d"
+        ),
+        "no_data": _("No data provided"),
+        "value_required": _("Value is required"),
+        "not_enough_cols": _("At least %d columns must be filled"),
+        "regex_not_match": _("The regexp doesn't match."),
+        "improperly_configured": _(
             "Forced creation is set for model {} but this model is not in the "
-            "list of models allowed to be created."),
-        'does_not_exist_in_db': _("{} with values {} doesn't exist in the "
-          "database. Create it first or fix your source file."),
+            "list of models allowed to be created."
+        ),
+        "does_not_exist_in_db": _(
+            "{} with values {} doesn't exist in the "
+            "database. Create it first or fix your source file."
+        ),
     }
 
     def _create_models(self, force=False):
@@ -761,6 +843,7 @@ class Importer(object):
         Not useful anymore?
         """
         from ishtar_common import models
+
         q = models.ImporterType.objects.filter(slug=self.SLUG)
         if not force and (not self.SLUG or q.count()):
             return
@@ -768,45 +851,48 @@ class Importer(object):
             q.all()[0].delete()
         name = self.NAME if self.NAME else self.SLUG
 
-        model_name = self.OBJECT_CLS.__module__ + '.' + \
-            self.OBJECT_CLS.__name__
+        model_name = self.OBJECT_CLS.__module__ + "." + self.OBJECT_CLS.__name__
         model_cls, c = models.ImporterModel.object.get_or_create(
-            klass=model_name, default={'name': self.OBJECT_CLS.__name__}
+            klass=model_name, default={"name": self.OBJECT_CLS.__name__}
         )
 
-        unicity_keys = ''
+        unicity_keys = ""
         if self.UNICITY_KEYS:
             unicity_keys = ";".join(self.UNICITY_KEYS)
 
         importer = models.ImporterType.objects.create(
-            slug=self.SLUG, name=name, description=self.DESC,
-            associated_models=model_cls, unicity_keys=unicity_keys)
+            slug=self.SLUG,
+            name=name,
+            description=self.DESC,
+            associated_models=model_cls,
+            unicity_keys=unicity_keys,
+        )
 
         for default in self.DEFAULTS:
             values = self.DEFAULTS[default]
             imp_default = models.ImporterDefault.objects.create(
-                importer_type=importer,
-                target='__'.join(default))
+                importer_type=importer, target="__".join(default)
+            )
             for key in values:
-                if key in ('history_modifier',):
+                if key in ("history_modifier",):
                     continue
                 value = values[key]
-                if hasattr(value, 'txt_idx') and value.txt_idx:
+                if hasattr(value, "txt_idx") and value.txt_idx:
                     value = value.txt_idx
-                elif hasattr(value, 'pk') and value.pk:
+                elif hasattr(value, "pk") and value.pk:
                     value = value.pk
                 if callable(value):
                     value = value()
                 models.ImporterDefaultValues.objects.create(
-                    default_target=imp_default,
-                    target=key,
-                    value=value)
+                    default_target=imp_default, target=key, value=value
+                )
         for idx, line in enumerate(self.line_format):
             idx += 1
             if not line:
                 continue
             column = models.ImporterColumn.objects.create(
-                importer_type=importer, col_number=idx)
+                importer_type=importer, col_number=idx
+            )
             targets = line.field_name
             if type(targets) not in (list, tuple):
                 targets = [targets]
@@ -817,64 +903,73 @@ class Importer(object):
                 formater = formaters[idx_target]
                 formater_name = formater.__class__.__name__
                 if formater_name not in models.IMPORTER_TYPES_DCT:
-                    formater_name = 'UnknowType'
-                options = ''
-                if formater_name == 'TypeFormater':
-                    options = formater.model.__module__ + '.' + \
-                        formater.model.__name__
-                elif formater_name == 'UnicodeFormater':
-                    options = str(formater.max_length or '')
-                elif formater_name == 'DateFormater':
+                    formater_name = "UnknowType"
+                options = ""
+                if formater_name == "TypeFormater":
+                    options = formater.model.__module__ + "." + formater.model.__name__
+                elif formater_name == "UnicodeFormater":
+                    options = str(formater.max_length or "")
+                elif formater_name == "DateFormater":
                     options = formater.date_formats[0]
-                formater_model, created = \
-                    models.FormaterType.objects.get_or_create(
-                        formater_type=formater_name, options=options.strip(),
-                        many_split=getattr(formater, 'many_split', None) or '')
+                formater_model, created = models.FormaterType.objects.get_or_create(
+                    formater_type=formater_name,
+                    options=options.strip(),
+                    many_split=getattr(formater, "many_split", None) or "",
+                )
                 regexp_filter = None
-                if getattr(formater, 'regexp', None):
-                    regexp_filter, created = \
-                        models.Regexp.objects.get_or_create(
-                            regexp=formater.regex,
-                            defaults={'name': "Default name"})
+                if getattr(formater, "regexp", None):
+                    regexp_filter, created = models.Regexp.objects.get_or_create(
+                        regexp=formater.regex, defaults={"name": "Default name"}
+                    )
                 models.ImportTarget.objects.get_or_create(
-                    column=column, target=target, formater_type=formater_model,
-                    force_new=getattr(formater, 'force_new', False),
-                    concat=getattr(formater, 'concat', False),
-                    concat_str=getattr(formater, 'concat_str', ''),
+                    column=column,
+                    target=target,
+                    formater_type=formater_model,
+                    force_new=getattr(formater, "force_new", False),
+                    concat=getattr(formater, "concat", False),
+                    concat_str=getattr(formater, "concat_str", ""),
                     regexp_filter=regexp_filter,
-                    comment=line.comment)
+                    comment=line.comment,
+                )
         return True
 
     def _get_improperly_conf_error(self, model):
         from ishtar_common.models import ImporterModel
+
         cls_name = model.__module__ + "." + model.__name__
         q = ImporterModel.objects.filter(klass=cls_name)
         if q.count():
             cls_name = q.all()[0].name
-        return ImporterError(
-            str(self.ERRORS['improperly_configured']).format(cls_name))
+        return ImporterError(str(self.ERRORS["improperly_configured"]).format(cls_name))
 
     def _get_does_not_exist_in_db_error(self, model, data):
         from ishtar_common.models import ImporterModel
+
         cls_name = model.__module__ + "." + model.__name__
         q = ImporterModel.objects.filter(klass=cls_name)
         if q.count():
             cls_name = q.all()[0].name
-        values = ", ".join(
-            ["{}: {}".format(k, data[k]) for k in data]
-        )
+        values = ", ".join(["{}: {}".format(k, data[k]) for k in data])
         raise ImporterError(
-            str(self.ERRORS['does_not_exist_in_db']).format(cls_name, values))
+            str(self.ERRORS["does_not_exist_in_db"]).format(cls_name, values)
+        )
 
-    def __init__(self, skip_lines=0, reference_header=None,
-                 check_col_num=False, test=False, history_modifier=None,
-                 output='silent', import_instance=None,
-                 conservative_import=False):
+    def __init__(
+        self,
+        skip_lines=0,
+        reference_header=None,
+        check_col_num=False,
+        test=False,
+        history_modifier=None,
+        output="silent",
+        import_instance=None,
+        conservative_import=False,
+    ):
         """
-         * skip_line must be set if the data provided has got headers lines.
-         * a reference_header can be provided to perform a data compliance
-           check. It can be useful to warn about bad parsing.
-         * test doesn't write in the database
+        * skip_line must be set if the data provided has got headers lines.
+        * a reference_header can be provided to perform a data compliance
+          check. It can be useful to warn about bad parsing.
+        * test doesn't write in the database
         """
         self.skip_lines = skip_lines
         self.reference_header = reference_header
@@ -918,14 +1013,15 @@ class Importer(object):
                 self.history_modifier = self.import_instance.user.user_ptr
             else:
                 # import made by the CLI: get the first admin
-                self.history_modifier = User.objects.filter(
-                    is_superuser=True).order_by('pk')[0]
+                self.history_modifier = User.objects.filter(is_superuser=True).order_by(
+                    "pk"
+                )[0]
 
     def post_processing(self, idx_line, item):
         # force django based post-processing for the item
         item = item.__class__.objects.get(pk=item.pk)
         item.save()
-        if hasattr(item, 'RELATED_POST_PROCESS'):
+        if hasattr(item, "RELATED_POST_PROCESS"):
             for related_key in item.RELATED_POST_PROCESS:
                 for related in getattr(item, related_key).all():
                     related.save()
@@ -937,8 +1033,7 @@ class Importer(object):
                 self.errors.append((idx_line, None, msg))
         return item
 
-    def initialize(self, table, output='silent', choose_default=False,
-                   user=None):
+    def initialize(self, table, output="silent", choose_default=False, user=None):
         """
         copy vals in columns and initialize formaters
         * output:
@@ -948,7 +1043,7 @@ class Importer(object):
            (further exploitation by web interface)
          - user: associated user
         """
-        assert output in ('silent', 'cli', 'db')
+        assert output in ("silent", "cli", "db")
         vals = []
         for idx_line, line in enumerate(table):
             if self.skip_lines > idx_line:
@@ -969,22 +1064,34 @@ class Importer(object):
                     db_targets = []
                     for field_name in field_names:
                         db_targets.append(
-                            self.DB_TARGETS["{}-{}".format(
-                                idx + 1, field_name)])
+                            self.DB_TARGETS["{}-{}".format(idx + 1, field_name)]
+                        )
                     formater.reinit_db_target(db_targets, user=user)
 
-                formater.init(vals[idx], output, choose_default=choose_default,
-                              import_instance=self.import_instance,
-                              user=user)
+                formater.init(
+                    vals[idx],
+                    output,
+                    choose_default=choose_default,
+                    import_instance=self.import_instance,
+                    user=user,
+                )
 
     def get_formaters(self):
         return self.line_format
 
-    def importation(self, table, initialize=True, choose_default=False,
-                    user=None, line_to_process=None, simulate=False):
+    def importation(
+        self,
+        table,
+        initialize=True,
+        choose_default=False,
+        user=None,
+        line_to_process=None,
+        simulate=False,
+    ):
         if initialize:
-            self.initialize(table, self.output,
-                            choose_default=choose_default, user=user)
+            self.initialize(
+                table, self.output, choose_default=choose_default, user=user
+            )
         self.simulate = simulate
         return self._importation(table, line_to_process=line_to_process)
 
@@ -996,13 +1103,14 @@ class Importer(object):
             return
         self.DB_TARGETS = {}
         from ishtar_common.models import ImporterColumn, ImportTarget
+
         for idx, line in enumerate(self.line_format):
             idx += 1
             if not line:
                 continue
             col = ImporterColumn.objects.get(
-                importer_type=self.import_instance.importer_type,
-                col_number=idx)
+                importer_type=self.import_instance.importer_type, col_number=idx
+            )
             formater = line.formater
             targets = line.field_name
             if type(formater) not in (list, tuple):
@@ -1012,18 +1120,26 @@ class Importer(object):
                 tg = target
                 if type(target) == list and type(target[0]) == list:
                     tg = target[0]
-                self.DB_TARGETS["{}-{}".format(idx, tg)] = \
-                    ImportTarget.objects.get(column=col, target=tg)
+                self.DB_TARGETS["{}-{}".format(idx, tg)] = ImportTarget.objects.get(
+                    column=col, target=tg
+                )
 
     @classmethod
     def _field_name_to_data_dict(
-            cls, field_name, value, data, force_value=False, concat=False,
-            concat_str="", force_new=False):
+        cls,
+        field_name,
+        value,
+        data,
+        force_value=False,
+        concat=False,
+        concat_str="",
+        force_new=False,
+    ):
         field_names = field_name
         if type(field_names) not in (list, tuple):
             field_names = [field_name]
         for field_name in field_names:
-            keys = field_name.split('__')
+            keys = field_name.split("__")
             current_data = data
             for idx, key in enumerate(keys):
                 if idx == (len(keys) - 1):  # last
@@ -1040,23 +1156,26 @@ class Importer(object):
                             if not value:
                                 continue
                             current_data[key] = (
-                                current_data[key] + (concat_str or "")) \
-                                if current_data[key] else ""
+                                (current_data[key] + (concat_str or ""))
+                                if current_data[key]
+                                else ""
+                            )
                             current_data[key] += value
                     elif force_value and value:
-                        if concat_str and key in current_data \
-                                and current_data[key]:
-                            current_data[key] = str(current_data[key]) + \
-                                concat_str + str(value)
+                        if concat_str and key in current_data and current_data[key]:
+                            current_data[key] = (
+                                str(current_data[key]) + concat_str + str(value)
+                            )
                         else:
                             current_data[key] = value
                     elif key not in current_data or not current_data[key]:
                         current_data[key] = value
                     elif concat_str:
-                        current_data[key] = str(current_data[key]) +\
-                            concat_str + str(value)
+                        current_data[key] = (
+                            str(current_data[key]) + concat_str + str(value)
+                        )
                     if force_new:
-                        current_data['__force_new'] = True
+                        current_data["__force_new"] = True
                 elif key not in current_data:
                     current_data[key] = {}
                 current_data = current_data[key]
@@ -1066,10 +1185,12 @@ class Importer(object):
         self.match_table = {}
         table = list(table)
         if not table or not table[0]:
-            raise ImporterError(self.ERRORS['no_data'], ImporterError.HEADER)
+            raise ImporterError(self.ERRORS["no_data"], ImporterError.HEADER)
         if self.check_col_num and len(table[0]) > len(self.line_format):
-            raise ImporterError(self.ERRORS['too_many_cols'] % {
-                'user_col': len(table[0]), 'ref_col': len(self.line_format)})
+            raise ImporterError(
+                self.ERRORS["too_many_cols"]
+                % {"user_col": len(table[0]), "ref_col": len(self.line_format)}
+            )
         self.errors = []
         self.validity = []
         self.number_imported = 0
@@ -1083,15 +1204,16 @@ class Importer(object):
         # min col number to be filled
         self.min_col_number = len(self.line_format) - idx_last_col
         # check the conformity with the reference header
-        if self.reference_header and \
-           self.skip_lines and \
-           self.reference_header != table[0]:
-            raise ImporterError(self.ERRORS['header_check'],
-                                type=ImporterError.HEADER)
+        if (
+            self.reference_header
+            and self.skip_lines
+            and self.reference_header != table[0]
+        ):
+            raise ImporterError(self.ERRORS["header_check"], type=ImporterError.HEADER)
         self.now = datetime.datetime.now()
         start = datetime.datetime.now()
         total = len(table)
-        if self.output == 'cli':
+        if self.output == "cli":
             sys.stdout.write("\n")
         results = []
         for idx_line, line in enumerate(table):
@@ -1101,7 +1223,7 @@ class Importer(object):
                     continue
                 if idx_line > line_to_process:
                     return results
-            if self.output == 'cli':
+            if self.output == "cli":
                 left = None
                 if idx_line > 10:
                     ellapsed = datetime.datetime.now() - start
@@ -1111,7 +1233,7 @@ class Importer(object):
                 txt = "\r* %d/%d" % (idx_line + 1, total)
                 if left:
                     txt += " (%d seconds left)" % left
-                sys.stdout.write(txt.encode('utf-8'))
+                sys.stdout.write(txt.encode("utf-8"))
                 sys.stdout.flush()
             try:
                 results.append(self._line_processing(idx_line, line))
@@ -1132,8 +1254,11 @@ class Importer(object):
         if not line:
             self.validity.append([])
             return
-        if not self.simulate and self.import_instance and \
-                not self.import_instance.has_changes(idx_line):
+        if (
+            not self.simulate
+            and self.import_instance
+            and not self.import_instance.has_changes(idx_line)
+        ):
             self.validity.append(line)
             return
 
@@ -1148,8 +1273,7 @@ class Importer(object):
             self.current_csv_line = line
 
         n = datetime.datetime.now()
-        logger.debug('%s - Processing line %d' % (str(n - self.now),
-                                                  idx_line))
+        logger.debug("%s - Processing line %d" % (str(n - self.now), idx_line))
         self.now = n
         n2 = n
         self.c_errors = False
@@ -1163,19 +1287,23 @@ class Importer(object):
         self.validity.append(c_row)
         if not self.c_errors and (idx_col + 1) < self.min_col_number:
             self.c_errors = True
-            self.errors.append((
-                idx_line + 1, idx_col + 1,
-                self.ERRORS['not_enough_cols'] % self.min_col_number))
+            self.errors.append(
+                (
+                    idx_line + 1,
+                    idx_col + 1,
+                    self.ERRORS["not_enough_cols"] % self.min_col_number,
+                )
+            )
         if self.c_errors:
             return
         n = datetime.datetime.now()
-        logger.debug('* %s - Cols read' % (str(n - n2)))
+        logger.debug("* %s - Cols read" % (str(n - n2)))
         n2 = n
         if self.test:
             return
         # manage unicity of items (mainly for updates)
-        if 'history_modifier' in get_all_field_names(self.OBJECT_CLS):
-            data['history_modifier'] = self.history_modifier
+        if "history_modifier" in get_all_field_names(self.OBJECT_CLS):
+            data["history_modifier"] = self.history_modifier
 
         self.new_objects, self.updated_objects = [], []
         self.ambiguous_objects, self.not_find_objects = [], []
@@ -1189,8 +1317,7 @@ class Importer(object):
         if self.import_instance:
             self.import_instance.add_imported_line(self.idx_line)
 
-        if self.import_instance and hasattr(obj, 'imports') \
-           and created:
+        if self.import_instance and hasattr(obj, "imports") and created:
             obj.imports.add(self.import_instance)
 
         if created:
@@ -1198,17 +1325,18 @@ class Importer(object):
         else:
             self.number_updated += 1
 
-        if not created and 'defaults' in data:
-            for k in data['defaults']:
-                setattr(obj, k, data['defaults'][k])
+        if not created and "defaults" in data:
+            for k in data["defaults"]:
+                setattr(obj, k, data["defaults"][k])
             obj.save()
         n = datetime.datetime.now()
-        logger.debug('* %s - Item saved' % (str(n - n2)))
+        logger.debug("* %s - Item saved" % (str(n - n2)))
         n2 = n
         for formater, value in self._throughs:
             n = datetime.datetime.now()
-            logger.debug('* %s - Processing formater %s' % (str(n - n2),
-                         formater.field_name))
+            logger.debug(
+                "* %s - Processing formater %s" % (str(n - n2), formater.field_name)
+            )
             n2 = n
             data = {}
             if formater.through_dict:
@@ -1218,38 +1346,41 @@ class Importer(object):
             data[formater.field_name] = value
             through_cls = formater.through
             if formater.through_unicity_keys:
-                data['defaults'] = {}
+                data["defaults"] = {}
                 for k in list(data.keys()):
-                    if k not in formater.through_unicity_keys \
-                       and k != 'defaults':
-                        data['defaults'][k] = data.pop(k)
+                    if k not in formater.through_unicity_keys and k != "defaults":
+                        data["defaults"][k] = data.pop(k)
             created = False
-            if '__force_new' in data:
-                if self.MODEL_CREATION_LIMIT and \
-                                through_cls not in self.MODEL_CREATION_LIMIT:
+            if "__force_new" in data:
+                if (
+                    self.MODEL_CREATION_LIMIT
+                    and through_cls not in self.MODEL_CREATION_LIMIT
+                ):
                     raise self._get_improperly_conf_error(through_cls)
-                created = data.pop('__force_new')
+                created = data.pop("__force_new")
                 t_obj = through_cls.objects.create(**data)
             else:
-                if not self.MODEL_CREATION_LIMIT or \
-                        through_cls in self.MODEL_CREATION_LIMIT:
+                if (
+                    not self.MODEL_CREATION_LIMIT
+                    or through_cls in self.MODEL_CREATION_LIMIT
+                ):
                     t_obj, created = through_cls.objects.get_or_create(**data)
                 else:
                     get_data = data.copy()
-                    if 'defaults' in get_data:
-                        get_data.pop('defaults')
+                    if "defaults" in get_data:
+                        get_data.pop("defaults")
                     try:
                         t_obj = through_cls.objects.get(**get_data)
                     except through_cls.DoesNotExist:
                         raise self._get_does_not_exist_in_db_error(
-                            through_cls, get_data)
-            if not created and 'defaults' in data:
+                            through_cls, get_data
+                        )
+            if not created and "defaults" in data:
                 t_obj = t_obj.__class__.objects.get(pk=t_obj.pk)
-                for k in data['defaults']:
-                    setattr(t_obj, k, data['defaults'][k])
+                for k in data["defaults"]:
+                    setattr(t_obj, k, data["defaults"][k])
                 t_obj.save()
-            if self.import_instance and hasattr(t_obj, 'imports') \
-               and created:
+            if self.import_instance and hasattr(t_obj, "imports") and created:
                 t_obj.imports.add(self.import_instance)
         if not obj:
             return data
@@ -1270,35 +1401,37 @@ class Importer(object):
             self._post_processing.append((formater, val))
 
         if not formater or not formater.field_name:
-            c_row.append(_('Not imported'))
+            c_row.append(_("Not imported"))
             return
 
         if formater.regexp:
             # multiline regexp is a mess...
-            val = val.replace('\n', NEW_LINE_BREAK)
+            val = val.replace("\n", NEW_LINE_BREAK)
             match = formater.regexp.match(val)
             if not match:
                 if formater.required:
                     self.errors.append(
-                        (idx_line + 1, idx_col + 1,
-                         self.ERRORS['value_required']))
+                        (idx_line + 1, idx_col + 1, self.ERRORS["value_required"])
+                    )
                     self.c_errors = True
                 elif not val.strip():
                     c_row.append("")
                     return
-                val = val.replace(NEW_LINE_BREAK, '\n')
+                val = val.replace(NEW_LINE_BREAK, "\n")
                 self.errors.append(
-                    (idx_line + 1, idx_col + 1,
-                     str(self.ERRORS['regex_not_match']) + val))
+                    (
+                        idx_line + 1,
+                        idx_col + 1,
+                        str(self.ERRORS["regex_not_match"]) + val,
+                    )
+                )
                 c_row.append("")
                 return
             val_group = []
             for g in formater.regexp.findall(val):
                 if isinstance(g, (tuple, list)):
                     g = "".join(g)
-                val_group.append(
-                    g.replace(NEW_LINE_BREAK, '\n') if g else ''
-                )
+                val_group.append(g.replace(NEW_LINE_BREAK, "\n") if g else "")
             val = "".join(val_group)
 
         field_names = formater.field_name
@@ -1314,7 +1447,7 @@ class Importer(object):
                 func = getattr(self, func)
 
             values = [val]
-            many_values = getattr(func, 'many_split', None)
+            many_values = getattr(func, "many_split", None)
             if many_values:
                 values = re.split(func.many_split, values[0])
                 # filter empty entries on m2m such as "my-value & "
@@ -1341,8 +1474,8 @@ class Importer(object):
             if self.DB_TARGETS:
                 formater.import_instance = self.import_instance
                 formater.reinit_db_target(
-                    self.DB_TARGETS["{}-{}".format(idx_col + 1, field_name)],
-                    idx_fields)
+                    self.DB_TARGETS["{}-{}".format(idx_col + 1, field_name)], idx_fields
+                )
             for idx, v in enumerate(values):
                 try:
                     """
@@ -1354,7 +1487,7 @@ class Importer(object):
                         value = func.format(*args)
                     else:
                     """
-                    if getattr(func, 'need_archive', False):
+                    if getattr(func, "need_archive", False):
                         value = func.format(v, archive=self.archive)
                     else:
                         value = func.format(v)
@@ -1362,7 +1495,7 @@ class Importer(object):
                     if formater.required:
                         self.c_errors = True
                     self.errors.append((idx_line + 1, idx_col + 1, str(e)))
-                    c_values.append('')
+                    c_values.append("")
                     return
                 if formater.value_format and value is not None and value != "":
                     value = formater.value_format.format(value)
@@ -1371,7 +1504,7 @@ class Importer(object):
                     # later
                     self.to_be_close.append(value)
                 formated_values.append(value)
-            if hasattr(func, 'match_table'):
+            if hasattr(func, "match_table"):
                 if field_name not in self.match_table:
                     self.match_table[field_name] = {}
                 self.match_table[field_name].update(func.match_table)
@@ -1390,8 +1523,9 @@ class Importer(object):
                 c_values.append(" ; ".join([str(v) for v in printed_values]))
             if value is None and formater.required:
                 self.c_errors = True
-                self.errors.append((idx_line + 1, idx_col + 1,
-                                   self.ERRORS['value_required']))
+                self.errors.append(
+                    (idx_line + 1, idx_col + 1, self.ERRORS["value_required"])
+                )
                 return
 
             field_names = [field_name]
@@ -1402,11 +1536,11 @@ class Importer(object):
                 # duplicate fields are only for the first occurrence
                 for duplicate_field in formater.duplicate_fields:
                     if type(duplicate_field[0]) in (list, tuple):
-                        duplicate_field, force_new, concat, conc_str = \
-                            duplicate_field[0]
+                        duplicate_field, force_new, concat, conc_str = duplicate_field[
+                            0
+                        ]
                     else:
-                        duplicate_field, force_new, concat, conc_str = \
-                            duplicate_field
+                        duplicate_field, force_new, concat, conc_str = duplicate_field
                     field_names += [duplicate_field]
                     force_news += [force_new]
                     concats += [concat]
@@ -1417,13 +1551,17 @@ class Importer(object):
             else:
                 for idx, f_name in enumerate(field_names):
                     self._field_name_to_data_dict(
-                        f_name, value, data, formater.force_value,
-                        force_new=force_news[idx], concat=concats[idx],
-                        concat_str=concat_str[idx])
+                        f_name,
+                        value,
+                        data,
+                        formater.force_value,
+                        force_new=force_news[idx],
+                        concat=concats[idx],
+                        concat_str=concat_str[idx],
+                    )
         c_row.append(" ; ".join([v for v in c_values]))
 
-    def _get_field_m2m(self, attribute, data, c_path, new_created,
-                       field_object):
+    def _get_field_m2m(self, attribute, data, c_path, new_created, field_object):
         """
         Manage and m2m field from raw data
 
@@ -1439,13 +1577,13 @@ class Importer(object):
 
         many_values = data.pop(attribute)
         model = None
-        if hasattr(field_object, 'rel'):
+        if hasattr(field_object, "rel"):
             model = field_object.rel.to
-        elif hasattr(field_object, 'related_model'):
+        elif hasattr(field_object, "related_model"):
             model = field_object.related_model
-        elif hasattr(field_object, 'to'):
+        elif hasattr(field_object, "to"):
             model = field_object.to
-        elif hasattr(field_object, 'model'):
+        elif hasattr(field_object, "model"):
             model = field_object.model
         if type(many_values) not in (list, tuple):
             many_values = [many_values]
@@ -1498,34 +1636,31 @@ class Importer(object):
 
             field_names = get_all_field_names(model)
             for v in vals:
-                if 'history_modifier' in field_names:
-                    if 'defaults' not in v:
-                        v['defaults'] = {}
-                    v['defaults']['history_modifier'] = \
-                        self.history_modifier
+                if "history_modifier" in field_names:
+                    if "defaults" not in v:
+                        v["defaults"] = {}
+                    v["defaults"]["history_modifier"] = self.history_modifier
                 m2m_m2ms = []
                 c_c_path = c_path[:]
                 for k in list(v.keys()):
                     if k not in field_names:
                         continue
-                    self.get_field(model, k, v, m2m_m2ms, c_c_path,
-                                   new_created)
-                if '__force_new' in v:
-                    created = v.pop('__force_new')
-                    key = ";".join(["{}-{}".format(k, v[k])
-                                    for k in sorted(v.keys())])
+                    self.get_field(model, k, v, m2m_m2ms, c_c_path, new_created)
+                if "__force_new" in v:
+                    created = v.pop("__force_new")
+                    key = ";".join(["{}-{}".format(k, v[k]) for k in sorted(v.keys())])
                     # only one forced creation
-                    if attribute in new_created \
-                            and key in new_created[attribute]:
+                    if attribute in new_created and key in new_created[attribute]:
                         continue
                     if attribute not in new_created:
                         new_created[attribute] = []
                     new_created[attribute].append(key)
-                    has_values = bool([1 for k in v
-                                       if v[k] and k != "defaults"])
+                    has_values = bool([1 for k in v if v[k] and k != "defaults"])
                     if has_values:
-                        if self.MODEL_CREATION_LIMIT and \
-                                        model not in self.MODEL_CREATION_LIMIT:
+                        if (
+                            self.MODEL_CREATION_LIMIT
+                            and model not in self.MODEL_CREATION_LIMIT
+                        ):
                             raise self._get_improperly_conf_error(model)
                         if "defaults" in v:
                             default_values = v.pop("defaults")
@@ -1536,7 +1671,7 @@ class Importer(object):
                     else:
                         continue
                 else:
-                    v['defaults'] = v.get('defaults', {})
+                    v["defaults"] = v.get("defaults", {})
                     extra_fields = {}
                     # "File" type is a temp object and can be different
                     # for the same filename - it must be treated
@@ -1544,33 +1679,32 @@ class Importer(object):
                     for field in model._meta.fields:
                         k = field.name
                         # attr_class is a FileField attribute
-                        if hasattr(field, 'attr_class') and k in v:
+                        if hasattr(field, "attr_class") and k in v:
                             extra_fields[k] = v.pop(k)
                     created = False
-                    if not self.MODEL_CREATION_LIMIT or \
-                            model in self.MODEL_CREATION_LIMIT:
+                    if (
+                        not self.MODEL_CREATION_LIMIT
+                        or model in self.MODEL_CREATION_LIMIT
+                    ):
                         try:
-                            v, created = model.objects.get_or_create(
-                                **v)
+                            v, created = model.objects.get_or_create(**v)
                         except FieldError as e:
                             raise ImporterError(
-                                str(
-                                    _("Importer configuration error: "
-                                      "\"{}\".")).format(e))
+                                str(_("Importer configuration error: " '"{}".')).format(
+                                    e
+                                )
+                            )
                         except Exception as e:
-                            msg = str(
-                                _("Import error: {} - \"{}\".")
-                            ).format(model, e)
+                            msg = str(_('Import error: {} - "{}".')).format(model, e)
                             raise ImporterError(msg)
                     else:
                         get_v = v.copy()
-                        if 'defaults' in get_v:
-                            get_v.pop('defaults')
+                        if "defaults" in get_v:
+                            get_v.pop("defaults")
                         try:
                             v = model.objects.get(**get_v)
                         except model.DoesNotExist:
-                            raise self._get_does_not_exist_in_db_error(
-                                model, get_v)
+                            raise self._get_does_not_exist_in_db_error(model, get_v)
                     changed = False
                     for k in extra_fields.keys():
                         if extra_fields[k]:
@@ -1583,8 +1717,7 @@ class Importer(object):
                         objs = [objs]
                     for obj in objs:
                         getattr(v, att).add(obj)
-                if self.import_instance \
-                        and hasattr(v, 'imports') and created:
+                if self.import_instance and hasattr(v, "imports") and created:
                     v.imports.add(self.import_instance)
                 m2ms.append((attribute, v))
         return m2ms
@@ -1600,15 +1733,16 @@ class Importer(object):
         :return: None
         """
         func = getattr(cls, attribute)
-        if func.importer_trigger == 'pre':
+        if func.importer_trigger == "pre":
             func(data, data[attribute])
-        elif func.importer_trigger == 'post':
-            self._item_post_processing.append([attribute, data,
-                                               data[attribute]])
+        elif func.importer_trigger == "post":
+            self._item_post_processing.append([attribute, data, data[attribute]])
         else:
-            logger.warning("Unknow importer_trigger '{}' for '{}'".format(
-                func.importer_trigger, attribute
-            ))
+            logger.warning(
+                "Unknow importer_trigger '{}' for '{}'".format(
+                    func.importer_trigger, attribute
+                )
+            )
         data.pop(attribute)
 
     def get_field(self, cls, attribute, data, m2ms, c_path, new_created):
@@ -1624,53 +1758,62 @@ class Importer(object):
         multiple creation
         :return: None
         """
-        if hasattr(cls, attribute) and \
-                getattr(getattr(cls, attribute), 'importer_trigger', None):
+        if hasattr(cls, attribute) and getattr(
+            getattr(cls, attribute), "importer_trigger", None
+        ):
             # importer trigger
             self._set_importer_trigger(cls, attribute, data)
             return
-        if attribute == 'data':  # json field
+        if attribute == "data":  # json field
             # no need to do anything
             return
-        if attribute == 'get_default':
+        if attribute == "get_default":
             # force evaluation of default value for this field
             return
         try:
             field_object = cls._meta.get_field(attribute)
         except FieldDoesNotExist:
-            raise ImporterError(str(
-                _("Importer configuration error: field \"{}\" does not exist "
-                  "for {}.")).format(attribute, cls._meta.verbose_name))
+            raise ImporterError(
+                str(
+                    _(
+                        'Importer configuration error: field "{}" does not exist '
+                        "for {}."
+                    )
+                ).format(attribute, cls._meta.verbose_name)
+            )
         if field_object.many_to_many:
             try:
-                m2ms += self._get_field_m2m(attribute, data, c_path,
-                                            new_created, field_object)
+                m2ms += self._get_field_m2m(
+                    attribute, data, c_path, new_created, field_object
+                )
             except Exception as e:
                 self.errors.append((self.idx_line, None, str(e)))
             return
-        if not hasattr(field_object, 'rel') or not field_object.rel:
+        if not hasattr(field_object, "rel") or not field_object.rel:
             return
         if type(data[attribute]) == list:
             # extract the first item from list
             # be careful if the list has more than one item this is arbitrary
             if len(data[attribute]) > 1:
                 logger.warning(
-                    'Import {}: {} has many when only one is expected. Get '
-                    'the first one but it is not OK!'.format(
-                     self.import_instance, attribute))
+                    "Import {}: {} has many when only one is expected. Get "
+                    "the first one but it is not OK!".format(
+                        self.import_instance, attribute
+                    )
+                )
             data[attribute] = data[attribute][0]
             return
         if not isinstance(data[attribute], dict):
             # we treat only dict formated values
             return
         # put history_modifier for every created item
-        if 'history_modifier' in get_all_field_names(field_object.rel.to):
-            data[attribute]['history_modifier'] = \
-                self.history_modifier
+        if "history_modifier" in get_all_field_names(field_object.rel.to):
+            data[attribute]["history_modifier"] = self.history_modifier
         try:
             c_path.append(attribute)
             data[attribute], created = self.get_object(
-                field_object.rel.to, data[attribute].copy(), c_path)
+                field_object.rel.to, data[attribute].copy(), c_path
+            )
         except ImporterError as msg:
             self.errors.append((self.idx_line, None, msg))
             data[attribute] = None
@@ -1684,8 +1827,8 @@ class Importer(object):
             return data, False
 
         is_empty = not bool(
-            [k for k in data if k not in ('history_modifier', 'defaults')
-                and data[k]])
+            [k for k in data if k not in ("history_modifier", "defaults") and data[k]]
+        )
         if is_empty:
             # if no value, no creation
             return None, False
@@ -1704,29 +1847,29 @@ class Importer(object):
                     continue
                 if not data[attribute]:
                     if hasattr(cls, attribute) and getattr(
-                            getattr(cls, attribute), 'importer_trigger', None):
+                        getattr(cls, attribute), "importer_trigger", None
+                    ):
                         data.pop(attribute)
                         continue
                     field_object = cls._meta.get_field(attribute)
                     if field_object.many_to_many:
                         data.pop(attribute)
                     continue
-                if attribute != '__force_new':
-                    self.get_field(cls, attribute, data, m2ms, c_c_path,
-                                   new_created)
+                if attribute != "__force_new":
+                    self.get_field(cls, attribute, data, m2ms, c_c_path, new_created)
         except (ValueError, IntegrityError, FieldDoesNotExist) as e:
             try:
                 message = str(e)
             except (UnicodeDecodeError, UnicodeDecodeError):
-                message = ''
+                message = ""
             try:
                 data = str(data)
             except UnicodeDecodeError:
-                data = ''
+                data = ""
             raise ImporterError(
                 "Erreur d'import %s %s, contexte : %s, erreur : %s"
-                % (str(cls), str("__".join(path)),
-                   str(data), message))
+                % (str(cls), str("__".join(path)), str(data), message)
+            )
 
         # image field is not serialized
         image, associated_file = None, None
@@ -1744,7 +1887,7 @@ class Importer(object):
 
         for k in list(create_dict.keys()):
             # filter unnecessary default values but not the json field
-            if isinstance(create_dict[k], dict) and k != 'data':
+            if isinstance(create_dict[k], dict) and k != "data":
                 if self.simulate:
                     create_dict[k] = _("* created *")
                 else:
@@ -1763,10 +1906,8 @@ class Importer(object):
                 if (k not in data or not data[k]) and self._defaults[path][k]:
                     defaults[k] = self._defaults[path][k]
 
-        if 'history_modifier' in create_dict:
-            defaults.update({
-                'history_modifier': create_dict.pop('history_modifier')
-            })
+        if "history_modifier" in create_dict:
+            defaults.update({"history_modifier": create_dict.pop("history_modifier")})
 
         created = False
         post_save_keys = []
@@ -1782,14 +1923,16 @@ class Importer(object):
                         if getattr(dct[key], "post_save", True):
                             dct.pop(key)
                             post_save_keys.append(key)
-                if '__force_new' in dct:
-                    created = dct.pop('__force_new')
+                if "__force_new" in dct:
+                    created = dct.pop("__force_new")
                     if not [k for k in dct if dct[k] is not None]:
                         return None, created
                     new_dct = defaults.copy()
                     new_dct.update(dct)
-                    if self.MODEL_CREATION_LIMIT and \
-                            cls not in self.MODEL_CREATION_LIMIT:
+                    if (
+                        self.MODEL_CREATION_LIMIT
+                        and cls not in self.MODEL_CREATION_LIMIT
+                    ):
                         raise self._get_improperly_conf_error(cls)
                     if not self.simulate:
                         obj = cls.objects.create(**new_dct)
@@ -1799,8 +1942,7 @@ class Importer(object):
                     # manage UNICITY_KEYS - only level 1
                     if not path and self.UNICITY_KEYS:
                         for k in list(dct.keys()):
-                            if k not in self.UNICITY_KEYS \
-                               and k != 'defaults':
+                            if k not in self.UNICITY_KEYS and k != "defaults":
                                 if dct[k]:
                                     defaults[k] = dct.pop(k)
                                 else:
@@ -1815,68 +1957,72 @@ class Importer(object):
                     if self.simulate:
                         q = cls.objects.filter(**dct)
                         if not q.count():
-                            if self.MODEL_CREATION_LIMIT and \
-                                    cls not in self.MODEL_CREATION_LIMIT:
-                                self.not_find_objects.append(
-                                    (path, cls, dct)
-                                )
+                            if (
+                                self.MODEL_CREATION_LIMIT
+                                and cls not in self.MODEL_CREATION_LIMIT
+                            ):
+                                self.not_find_objects.append((path, cls, dct))
                                 return _("* match not find *"), False
                             dct.update(defaults)
                             self.new_objects.append([path, cls, dct])
                             created = True
                         elif q.count() > 1:
-                            self.ambiguous_objects.append(
-                                (path, list(q.all()), dct)
-                            )
+                            self.ambiguous_objects.append((path, list(q.all()), dct))
                             if q.count() > 10:
-                                return _("* the query match more than 10 "
-                                         "results*"), False
+                                return (
+                                    _("* the query match more than 10 " "results*"),
+                                    False,
+                                )
                             else:
-                                return str(_(" or ")).join(
-                                    [str(item) for item in q.all()]
-                                ), False
+                                return (
+                                    str(_(" or ")).join(
+                                        [str(item) for item in q.all()]
+                                    ),
+                                    False,
+                                )
                         else:
-                            self.updated_objects.append(
-                                [path, q.all()[0], dct, {}])
-                            dct['defaults'] = defaults.copy()
+                            self.updated_objects.append([path, q.all()[0], dct, {}])
+                            dct["defaults"] = defaults.copy()
                     else:
                         if not dct and not defaults:
                             obj = None
                         else:
-                            if not self.MODEL_CREATION_LIMIT or \
-                                        cls in self.MODEL_CREATION_LIMIT:
-                                dct['defaults'] = defaults.copy()
+                            if (
+                                not self.MODEL_CREATION_LIMIT
+                                or cls in self.MODEL_CREATION_LIMIT
+                            ):
+                                dct["defaults"] = defaults.copy()
                                 obj, created = cls.objects.get_or_create(**dct)
                             else:
                                 try:
                                     obj = cls.objects.get(**dct)
-                                    dct['defaults'] = defaults.copy()
+                                    dct["defaults"] = defaults.copy()
                                 except cls.DoesNotExist:
-                                    raise self._get_does_not_exist_in_db_error(
-                                        cls, dct)
+                                    raise self._get_does_not_exist_in_db_error(cls, dct)
 
                     if not created and not path and self.UNICITY_KEYS:
                         updated_dct = {}
                         if self.conservative_import:
-                            for k in dct['defaults']:
-                                new_val = dct['defaults'][k]
-                                if new_val is None or new_val == '':
+                            for k in dct["defaults"]:
+                                new_val = dct["defaults"][k]
+                                if new_val is None or new_val == "":
                                     continue
                                 val = getattr(obj, k)
-                                if val is None or val == '':
+                                if val is None or val == "":
                                     updated_dct[k] = new_val
-                                elif k in self.concats \
-                                        and type(val) == str \
-                                        and type(new_val) == str:
+                                elif (
+                                    k in self.concats
+                                    and type(val) == str
+                                    and type(new_val) == str
+                                ):
                                     updated_dct[k] = val + "\n" + new_val
                         else:
-                            for k in dct['defaults']:
-                                new_val = dct['defaults'][k]
-                                if new_val is None or new_val == '':
+                            for k in dct["defaults"]:
+                                new_val = dct["defaults"][k]
+                                if new_val is None or new_val == "":
                                     continue
-                                if obj and k == 'data':
-                                    updated_dct[k] = update_data(obj.data,
-                                                                 new_val)
+                                if obj and k == "data":
+                                    updated_dct[k] = update_data(obj.data, new_val)
                                 else:
                                     updated_dct[k] = new_val
                         if updated_dct:
@@ -1886,16 +2032,19 @@ class Importer(object):
                                 for k in updated_dct:
                                     setattr(obj, k, updated_dct[k])
                                 obj.save()
-                if not self.simulate and self.import_instance and \
-                        hasattr(obj, 'imports') and created:
+                if (
+                    not self.simulate
+                    and self.import_instance
+                    and hasattr(obj, "imports")
+                    and created
+                ):
                     obj.imports.add(self.import_instance)
-            except (ValueError, IntegrityError, DatabaseError,
-                    GEOSException) as e:
+            except (ValueError, IntegrityError, DatabaseError, GEOSException) as e:
                 raise IntegrityError(str(e))
             except cls.MultipleObjectsReturned as e:
                 created = False
-                if 'defaults' in dct:
-                    dct.pop('defaults')
+                if "defaults" in dct:
+                    dct.pop("defaults")
                 raise IntegrityError(str(e))
                 # obj = cls.objects.filter(**dct).all()[0]
             for key in post_save_keys:
@@ -1914,13 +2063,15 @@ class Importer(object):
                     for v in values:
                         related_model = getattr(obj, attr)
                         # an intermediary model is used
-                        if hasattr(related_model, 'through') and \
-                                not related_model.through._meta.auto_created:
+                        if (
+                            hasattr(related_model, "through")
+                            and not related_model.through._meta.auto_created
+                        ):
                             # try to create it with default attributes
                             inter_model = related_model.through
                             target_name, item_name = None, None
                             for field in inter_model._meta.get_fields():
-                                rel_model = getattr(field, 'related_model', None)
+                                rel_model = getattr(field, "related_model", None)
                                 # assume that the first found is correct...
                                 if rel_model == v.__class__:
                                     target_name = field.name
@@ -1948,8 +2099,7 @@ class Importer(object):
                             current_data[item] = {}
                         current_data = current_data[item]
                     for key, value in m2ms:
-                        if not isinstance(value, list) and \
-                                not isinstance(value, tuple):
+                        if not isinstance(value, list) and not isinstance(value, tuple):
                             value = [value]
                         current_data[key] = value
 
@@ -1957,34 +2107,36 @@ class Importer(object):
                     return dct, True
                 else:
                     # defaults are not presented as matching data
-                    dct.pop('defaults')
+                    dct.pop("defaults")
                     return self.updated_objects[-1][1], False
 
             if m2ms:
                 # force post save script
                 obj.save()
-            if hasattr(obj, 'fix'):
+            if hasattr(obj, "fix"):
                 # post save/m2m specific fix
                 obj.fix()
         except IntegrityError as e:
             try:
                 message = str(e)
             except (UnicodeDecodeError, UnicodeDecodeError):
-                message = ''
+                message = ""
             try:
                 data = str(data)
             except UnicodeDecodeError:
-                data = ''
+                data = ""
             raise ImporterError(
                 "Erreur d'import %s %s, contexte : %s, erreur : %s"
-                % (str(cls), str("__".join(path)),
-                   str(data), message))
+                % (str(cls), str("__".join(path)), str(data), message)
+            )
         return obj, created
 
     def _format_csv_line(self, values, empty="-"):
-        return '"' + '","'.join(
-            [(v and str(v).replace('"', '""')) or empty
-             for v in values]) + '"'
+        return (
+            '"'
+            + '","'.join([(v and str(v).replace('"', '""')) or empty for v in values])
+            + '"'
+        )
 
     def _get_csv(self, rows, header=None, empty="-"):
         if not rows:
@@ -1999,14 +2151,13 @@ class Importer(object):
         return "\n".join(csv_v)
 
     def get_csv_errors(self):
-        return self._get_csv(
-            self.errors, header=[_("line"), _("col"), _("error")])
+        return self._get_csv(self.errors, header=[_("line"), _("col"), _("error")])
 
     def get_csv_result(self):
         return self._get_csv(self.validity)
 
     def get_csv_matches(self):
-        header = [_('field'), _('source'), _('result')]
+        header = [_("field"), _("source"), _("result")]
         values = []
         for field in self.match_table:
             for source in self.match_table[field]:
@@ -2022,10 +2173,12 @@ class Importer(object):
                 return
             if value not in choices_dct.values():
                 raise ValueError(
-                    _("\"%(value)s\" not in %(values)s") % {
-                        'value': value,
-                        'values': ", ".join(
-                            [val for val in choices_dct.values()])
-                    })
+                    _('"%(value)s" not in %(values)s')
+                    % {
+                        "value": value,
+                        "values": ", ".join([val for val in choices_dct.values()]),
+                    }
+                )
             return value
+
         return function