Flake8

1 files changed, 124 insertions, 92 deletions

diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index 992025bbb..a64b908ea 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -17,25 +17,30 @@
 
 # See the file COPYING for details.
 
-import copy, csv, datetime, logging, re, sys
-from tempfile import NamedTemporaryFile
+import copy
+import csv
+import datetime
+import io
+import logging
+import re
+import sys
 
 from django.contrib.auth.models import User
-from django.db import DatabaseError, IntegrityError, transaction
+from django.db import IntegrityError, transaction
 from django.template.defaultfilters import slugify
 from django.utils.translation import ugettext_lazy as _
 
-from ishtar_common.unicode_csv import UnicodeWriter
-
 NEW_LINE_BREAK = '#####@@@#####'
 
 RE_FILTER_CEDEX = re.compile("(.*) *(?: *CEDEX|cedex|Cedex|Cédex|cédex *\d*)")
 
+
 class ImportFormater(object):
     def __init__(self, field_name, formater=None, required=True, through=None,
-                through_key=None, through_dict=None, through_unicity_keys=None,
-                duplicate_fields=[], regexp=None, regexp_formater_args=[],
-                force_value=None, post_processing=False, concat=False, comment=""):
+                 through_key=None, through_dict=None,
+                 through_unicity_keys=None, duplicate_fields=[], regexp=None,
+                 regexp_formater_args=[], force_value=None,
+                 post_processing=False, concat=False, comment=""):
         self.field_name = field_name
         self.formater = formater
         self.required = required
@@ -63,7 +68,8 @@ class ImportFormater(object):
     def report_error(self, *args):
         return
 
-    def init(self, vals, output=None, choose_default=False, import_instance=None):
+    def init(self, vals, output=None, choose_default=False,
+             import_instance=None):
         try:
             lst = iter(self.formater)
         except TypeError:
@@ -77,9 +83,11 @@ class ImportFormater(object):
     def post_process(self, obj, context, value, owner=None):
         raise NotImplemented()
 
+
 class ImporterError(Exception):
     STANDARD = 'S'
     HEADER = 'H'
+
     def __init__(self, message, type='S'):
         self.msg = message
         self.type = type
@@ -87,6 +95,7 @@ class ImporterError(Exception):
     def __str__(self):
         return self.msg
 
+
 class Formater(object):
     def __init__(self, *args, **kwargs):
         self.db_target = kwargs.get('db_target', None)
@@ -98,6 +107,7 @@ class Formater(object):
               import_instance=None):
         return
 
+
 class ChoiceChecker(object):
     def report_new(self, comment):
         if not self.new_keys:
@@ -108,6 +118,7 @@ class ChoiceChecker(object):
             msg = u'"%s";"%s"\n' % (k, self.new_keys[k])
             sys.stderr.write(msg.encode('utf-8'))
 
+
 class UnicodeFormater(Formater):
     def __init__(self, max_length, clean=False, re_filter=None, notnull=False,
                  db_target=None):
@@ -135,13 +146,15 @@ class UnicodeFormater(Formater):
         except UnicodeDecodeError:
             return
         if len(value) > self.max_length:
-            raise ValueError(_(u"\"%(value)s\" is too long. "\
-                               u"The max length is %(length)d characters."
-                               ) % {'value':value, 'length':self.max_length})
+            raise ValueError(
+                _(u"\"%(value)s\" is too long. The max length is %(length)d "
+                  u"characters.") % {'value': value,
+                                     'length': self.max_length})
         if self.notnull and not value:
             return
         return value
 
+
 class BooleanFormater(Formater):
     def format(self, value):
         value = value.strip().upper()
@@ -150,7 +163,8 @@ class BooleanFormater(Formater):
         if value in ('', '0', 'NON', 'FAUX', 'NO', 'FALSE'):
             return False
         raise ValueError(_(u"\"%(value)s\" not equal to yes or no") % {
-                                                                 'value':value})
+            'value': value})
+
 
 class FloatFormater(Formater):
     def format(self, value):
@@ -161,7 +175,8 @@ class FloatFormater(Formater):
             return float(value)
         except ValueError:
             raise ValueError(_(u"\"%(value)s\" is not a float") % {
-                                                                 'value':value})
+                'value': value})
+
 
 class YearFormater(Formater):
     def format(self, value):
@@ -173,7 +188,8 @@ class YearFormater(Formater):
             assert value > 0 and value < (datetime.date.today().year + 30)
         except (ValueError, AssertionError):
             raise ValueError(_(u"\"%(value)s\" is not a valid date") % {
-                                                                 'value':value})
+                'value': value})
+
 
 class YearNoFuturFormater(Formater):
     def format(self, value):
@@ -185,7 +201,8 @@ class YearNoFuturFormater(Formater):
             assert value > 0 and value < (datetime.date.today().year)
         except (ValueError, AssertionError):
             raise ValueError(_(u"\"%(value)s\" is not a valid date") % {
-                                                                 'value':value})
+                'value': value})
+
 
 class IntegerFormater(Formater):
     def format(self, value):
@@ -196,7 +213,8 @@ class IntegerFormater(Formater):
             return int(value)
         except ValueError:
             raise ValueError(_(u"\"%(value)s\" is not an integer") % {
-                                                                 'value':value})
+                'value': value})
+
 
 class StrChoiceFormater(Formater, ChoiceChecker):
     def __init__(self, choices, strict=False, equiv_dict={}, model=None,
@@ -237,15 +255,15 @@ class StrChoiceFormater(Formater, ChoiceChecker):
 
     def _get_choices(self, comment=''):
         msgstr = comment + u" - "
-        msgstr += unicode(_(u"Choice for \"%s\" is not available. "\
-                           u"Which one is relevant?\n"))
+        msgstr += unicode(_(u"Choice for \"%s\" is not available. "
+                            u"Which one is relevant?\n"))
         idx = -1
         for idx, choice in enumerate(self.choices):
-            msgstr += u"%d. %s\n" % (idx+1, choice[1])
+            msgstr += u"%d. %s\n" % (idx + 1, choice[1])
         idx += 2
         if self.create:
             msgstr += unicode(_(u"%d. None of the above - create new")) % idx \
-                    + u"\n"
+                + u"\n"
             idx += 1
         msgstr += unicode(_(u"%d. None of the above - skip")) % idx + u"\n"
         return msgstr, idx
@@ -272,7 +290,7 @@ class StrChoiceFormater(Formater, ChoiceChecker):
             res = None
             if choose_default:
                 res = 1
-            while res not in range(1, idx+1):
+            while res not in range(1, idx + 1):
                 msg = msgstr % value
                 sys.stdout.write(msg.encode('utf-8'))
                 sys.stdout.write("\n>>> ")
@@ -299,8 +317,8 @@ class StrChoiceFormater(Formater, ChoiceChecker):
         if output == 'db' and self.db_target:
             from ishtar_common.models import TargetKey
             for missing in self.missings:
-                q = {'target':self.db_target, 'key':missing,
-                     'associated_import':import_instance}
+                q = {'target': self.db_target, 'key': missing,
+                     'associated_import': import_instance}
                 if TargetKey.objects.filter(**q).count():
                     continue
                 with transaction.commit_on_success():
@@ -324,6 +342,7 @@ class StrChoiceFormater(Formater, ChoiceChecker):
         if value in self.equiv_dict:
             return self.equiv_dict[value]
 
+
 class TypeFormater(StrChoiceFormater):
     def __init__(self, model, cli=False, defaults={}, many_split=False,
                  db_target=None):
@@ -359,6 +378,7 @@ class TypeFormater(StrChoiceFormater):
             values['order'] = order
         return self.model.objects.create(**values)
 
+
 class DateFormater(Formater):
     def __init__(self, date_formats=["%d/%m/%Y"], db_target=None):
         self.date_formats = date_formats
@@ -376,13 +396,14 @@ class DateFormater(Formater):
             except:
                 continue
         raise ValueError(_(u"\"%(value)s\" is not a valid date") % {
-                                                           'value':value})
+            'value': value})
+
 
 class StrToBoolean(Formater, ChoiceChecker):
     def __init__(self, choices={}, cli=False, strict=False, db_target=None):
         self.dct = copy.copy(choices)
         self.cli = cli
-        self.strict= strict
+        self.strict = strict
         self.db_target = db_target
         self.missings = set()
         if self.db_target:
@@ -412,8 +433,9 @@ class StrToBoolean(Formater, ChoiceChecker):
         if (not output or output == 'silent') and not choose_default:
             return
         msgstr = comment + u" - "
-        msgstr += unicode(_(u"Choice for \"%s\" is not available. "\
-                           u"Which one is relevant?\n"))
+        msgstr += unicode(_(
+            u"Choice for \"%s\" is not available. "
+            u"Which one is relevant?\n"))
         msgstr += u"1. True\n"
         msgstr += u"2. False\n"
         msgstr += u"3. Empty\n"
@@ -447,8 +469,8 @@ class StrToBoolean(Formater, ChoiceChecker):
             from ishtar_common.models import TargetKey
             for missing in self.missings:
                 try:
-                    q = {'target':self.db_target, 'key':missing,
-                         'associated_import':import_instance}
+                    q = {'target': self.db_target, 'key': missing,
+                         'associated_import': import_instance}
                     if not TargetKey.objects.filter(**q).count():
                         TargetKey.objects.create(**q)
                 except IntegrityError:
@@ -463,6 +485,7 @@ class StrToBoolean(Formater, ChoiceChecker):
 
 logger = logging.getLogger(__name__)
 
+
 def get_object_from_path(obj, path):
     for k in path.split('__')[:-1]:
         if not hasattr(obj, k):
@@ -470,6 +493,7 @@ def get_object_from_path(obj, path):
         obj = getattr(obj, k)
     return obj
 
+
 class Importer(object):
     DESC = ""
     LINE_FORMAT = []
@@ -480,21 +504,22 @@ class Importer(object):
     DEFAULTS = {}
     STR_CUT = {}
     ERRORS = {
-        'header_check':_(u"The given file is not correct. Check the file "
-                  u"format. If you use a CSV file: check that column separator "
-                  u"and encoding are similar to the ones used by the reference "
-                  u"file."),
-        'too_many_cols':_(u"Too many cols (%(user_col)d) when "
-                          u"maximum is %(ref_col)d"),
-        'no_data':_(u"No data provided"),
-        'value_required':_(u"Value is required"),
-        'not_enough_cols':_(u"At least %d columns must be filled"),
-        'regex_not_match':_(u"The regexp doesn't match.")
-        }
+        'header_check': _(
+            u"The given file is not correct. Check the file "
+            u"format. If you use a CSV file: check that column separator "
+            u"and encoding are similar to the ones used by the reference "
+            u"file."),
+        'too_many_cols': _(u"Too many cols (%(user_col)d) when "
+                           u"maximum is %(ref_col)d"),
+        'no_data': _(u"No data provided"),
+        'value_required': _(u"Value is required"),
+        'not_enough_cols': _(u"At least %d columns must be filled"),
+        'regex_not_match': _(u"The regexp doesn't match.")
+    }
 
     def __init__(self, skip_lines=0, reference_header=None,
-             check_col_num=False, test=False, history_modifier=None,
-             output='silent', import_instance=None):
+                 check_col_num=False, test=False, history_modifier=None,
+                 output='silent', import_instance=None):
         """
          * skip_line must be set if the data provided has got headers lines.
          * a reference_header can be provided to perform a data compliance
@@ -504,8 +529,8 @@ class Importer(object):
         self.skip_lines = skip_lines
         self.reference_header = reference_header
         self.test = test
-        self.errors = [] # list of (line, col, message)
-        self.validity = [] # list of (line, col, message)
+        self.errors = []  # list of (line, col, message)
+        self.validity = []  # list of (line, col, message)
         self.number_updated = 0
         self.number_created = 0
         self.check_col_num = check_col_num
@@ -526,7 +551,7 @@ class Importer(object):
             else:
                 # import made by the CLI: get the first admin
                 self.history_modifier = User.objects.filter(
-                                is_superuser=True).order_by('pk')[0]
+                    is_superuser=True).order_by('pk')[0]
 
     def post_processing(self, item, data):
         return item
@@ -537,8 +562,8 @@ class Importer(object):
         * output:
          - 'silent': no associations
          - 'cli': output by command line interface and stocked in the database
-         - 'db': output on the database with no interactive association (further
-           exploitation by web interface)
+         - 'db': output on the database with no interactive association
+           (further exploitation by web interface)
         """
         assert output in ('silent', 'cli', 'db')
         vals = []
@@ -571,12 +596,12 @@ class Importer(object):
             keys = field_name.split('__')
             current_data = data
             for idx, key in enumerate(keys):
-                if idx == (len(keys) - 1): # last
+                if idx == (len(keys) - 1):  # last
                     if concat:
                         if not value:
                             value = ""
                         current_data[key] = (current_data[key] + u"\n") or u""\
-                                            + value
+                            + value
                     elif force_value and value:
                         current_data[key] = value
                     elif key not in current_data or not current_data[key]:
@@ -592,7 +617,7 @@ class Importer(object):
             raise ImporterError(self.ERRORS['no_data'], ImporterError.HEADER)
         if self.check_col_num and len(table[0]) > len(self.line_format):
             raise ImporterError(self.ERRORS['too_many_cols'] % {
-                     'user_col':len(table[0]), 'ref_col':len(self.line_format)})
+                'user_col': len(table[0]), 'ref_col': len(self.line_format)})
         self.errors = []
         self.validity = []
         self.number_imported = 0
@@ -620,10 +645,10 @@ class Importer(object):
                 left = None
                 if idx_line > 10:
                     ellapsed = datetime.datetime.now() - start
-                    time_by_item = ellapsed/idx_line
+                    time_by_item = ellapsed / idx_line
                     if time_by_item:
-                        left = ((total - idx_line)*time_by_item).seconds
-                txt = u"\r* %d/%d" % (idx_line+1, total)
+                        left = ((total - idx_line) * time_by_item).seconds
+                txt = u"\r* %d/%d" % (idx_line + 1, total)
                 if left:
                     txt += u" (%d seconds left)" % left
                 sys.stdout.write(txt.encode('utf-8'))
@@ -640,8 +665,8 @@ class Importer(object):
         if not line:
             self.validity.append([])
             return
-        self._throughs = [] # list of (formater, value)
-        self._post_processing = [] # list of (formater, value)
+        self._throughs = []  # list of (formater, value)
+        self._post_processing = []  # list of (formater, value)
         data = {}
 
         # keep in database the raw line for testing purpose
@@ -652,7 +677,8 @@ class Importer(object):
             data[self.IMPORTED_LINE_FIELD] = output.getvalue()
 
         n = datetime.datetime.now()
-        logger.debug('%s - Processing line %d' % (unicode(n-self.now), idx_line))
+        logger.debug('%s - Processing line %d' % (unicode(n - self.now),
+                                                  idx_line))
         self.now = n
         n2 = n
         self.c_errors = False
@@ -666,12 +692,13 @@ class Importer(object):
         self.validity.append(c_row)
         if not self.c_errors and (idx_col + 1) < self.min_col_number:
             self.c_errors = True
-            self.errors.append((idx_line+1, idx_col+1,
-                          self.ERRORS['not_enough_cols'] % self.min_col_number))
+            self.errors.append((
+                idx_line + 1, idx_col + 1,
+                self.ERRORS['not_enough_cols'] % self.min_col_number))
         if self.c_errors:
             return
         n = datetime.datetime.now()
-        logger.debug('* %s - Cols read' % (unicode(n-n2)))
+        logger.debug('* %s - Cols read' % (unicode(n - n2)))
         n2 = n
         if self.test:
             return
@@ -684,7 +711,7 @@ class Importer(object):
                     data['defaults'][k] = data.pop(k)
 
         if 'history_modifier' in \
-                       self.OBJECT_CLS._meta.get_all_field_names():
+                self.OBJECT_CLS._meta.get_all_field_names():
             data['history_modifier'] = self.history_modifier
 
         obj, created = self.get_object(self.OBJECT_CLS, data)
@@ -702,12 +729,12 @@ class Importer(object):
                 setattr(obj, k, data['defaults'][k])
             obj.save()
         n = datetime.datetime.now()
-        logger.debug('* %s - Item saved' % (unicode(n-n2)))
+        logger.debug('* %s - Item saved' % (unicode(n - n2)))
         n2 = n
         for formater, value in self._throughs:
             n = datetime.datetime.now()
-            logger.debug('* %s - Processing formater %s' % (unicode(n-n2),
-                                                    formater.field_name))
+            logger.debug('* %s - Processing formater %s' % (unicode(n - n2),
+                         formater.field_name))
             n2 = n
             data = {}
             if formater.through_dict:
@@ -756,15 +783,17 @@ class Importer(object):
             match = formater.regexp.match(val)
             if not match:
                 if formater.required:
-                    self.errors.append((idx_line+1, idx_col+1,
-                                       self.ERRORS['value_required']))
+                    self.errors.append(
+                        (idx_line + 1, idx_col + 1,
+                         self.ERRORS['value_required']))
                 elif not val.strip():
                     c_row.append("")
                     return
                 self.c_errors = True
                 val = val.replace(NEW_LINE_BREAK, '\n')
-                self.errors.append((idx_line+1, idx_col+1,
-                         unicode(self.ERRORS['regex_not_match']) + val))
+                self.errors.append(
+                    (idx_line + 1, idx_col + 1,
+                     unicode(self.ERRORS['regex_not_match']) + val))
                 c_row.append("")
                 return
             val_group = [v.replace(NEW_LINE_BREAK, '\n')
@@ -798,7 +827,7 @@ class Importer(object):
                 except ValueError, e:
                     if formater.required:
                         self.c_errors = True
-                    self.errors.append((idx_line+1, idx_col+1, e.message))
+                    self.errors.append((idx_line + 1, idx_col + 1, e.message))
                     c_values.append('')
                     return
                 formated_values.append(value)
@@ -814,10 +843,11 @@ class Importer(object):
                 # don't reunicode - unicoded values
                 c_values.append(u" ; ".join([v for v in printed_values]))
             except TypeError:
-                c_values.append(u" ; ".join([unicode(v) for v in printed_values]))
-            if value == None and formater.required:
+                c_values.append(u" ; ".join([unicode(v)
+                                             for v in printed_values]))
+            if value is None and formater.required:
                 self.c_errors = True
-                self.errors.append((idx_line+1, idx_col+1,
+                self.errors.append((idx_line + 1, idx_col + 1,
                                    self.ERRORS['value_required']))
                 return
 
@@ -834,8 +864,8 @@ class Importer(object):
                 self._throughs.append((formater, value))
             else:
                 for field_name in field_names:
-                    self._field_name_to_data_dict(field_name,
-                              value, data, formater.force_value)
+                    self._field_name_to_data_dict(
+                        field_name, value, data, formater.force_value)
         c_row.append(u" ; ".join([v for v in c_values]))
 
     def get_object(self, cls, data, path=[]):
@@ -846,7 +876,7 @@ class Importer(object):
                 if not data[attribute]:
                     continue
                 field_object, model, direct, m2m = \
-                                    cls._meta.get_field_by_name(attribute)
+                    cls._meta.get_field_by_name(attribute)
                 if m2m:
                     many_values = data.pop(attribute)
                     if hasattr(field_object, 'rel'):
@@ -865,12 +895,12 @@ class Importer(object):
 
                             # contruct many dict for each values
                             default_dict = {}
-                            ## init with simple values that will be duplicated
+                            # # init with simple values that will be duplicated
                             for key in val.keys():
                                 if type(val[key]) not in (list, tuple):
                                     default_dict[key] = val[key]
                             vals.append(default_dict.copy())
-                            ## manage multiple values
+                            # # manage multiple values
                             for key in val.keys():
                                 if type(val[key]) in (list, tuple):
                                     for idx, v in enumerate(val[key]):
@@ -893,23 +923,23 @@ class Importer(object):
                                 if 'history_modifier' in \
                                         model._meta.get_all_field_names():
                                     v['defaults']['history_modifier'] = \
-                                                          self.history_modifier
+                                        self.history_modifier
                                     v, created = model.objects.get_or_create(
-                                                                            **v)
+                                        **v)
                                     if self.import_instance \
                                        and hasattr(v, 'imports') and created:
                                         v.imports.add(self.import_instance)
                                     m2ms.append((attribute, v))
                 elif hasattr(field_object, 'rel') and field_object.rel and \
-                   type(data[attribute]) == dict:
+                        type(data[attribute]) == dict:
                     c_path.append(attribute)
                     # put history_modifier for every created item
                     if 'history_modifier' in \
                        field_object.rel.to._meta.get_all_field_names():
                         data[attribute]['history_modifier'] = \
-                                                    self.history_modifier
+                            self.history_modifier
                     data[attribute], created = self.get_object(
-                                   field_object.rel.to, data[attribute], c_path)
+                        field_object.rel.to, data[attribute], c_path)
             # default values
             path = tuple(path)
             if path in self._defaults:
@@ -931,7 +961,8 @@ class Importer(object):
             defaults = {}
             if 'history_modifier' in create_dict:
                 defaults = {
-                        'history_modifier':create_dict.pop('history_modifier')}
+                    'history_modifier': create_dict.pop('history_modifier')
+                }
             try:
                 try:
                     dct = create_dict.copy()
@@ -954,7 +985,7 @@ class Importer(object):
             except IntegrityError as e:
                 message = e.message
                 try:
-                    message = unicode(e.message.decode('utf-8'))
+                    message = e.message.decode('utf-8')
                 except (UnicodeDecodeError, UnicodeDecodeError):
                     message = ''
                 try:
@@ -962,8 +993,8 @@ class Importer(object):
                 except UnicodeDecodeError:
                     data = ''
                 raise ImporterError(
-                    "Erreur d'import %s, contexte : %s, erreur : %s" \
-                    % (unicode(cls), unicode(data), e.message.decode('utf-8')))
+                    "Erreur d'import %s, contexte : %s, erreur : %s"
+                    % (unicode(cls), unicode(data), message))
             return obj, created
         return data
 
@@ -982,8 +1013,8 @@ class Importer(object):
         return u"\n".join(csv_v)
 
     def get_csv_errors(self):
-        return self._get_csv(self.errors,
-                            header=[_("line"), _("col"), _("error")])
+        return self._get_csv(
+            self.errors, header=[_("line"), _("col"), _("error")])
 
     def get_csv_result(self):
         return self._get_csv(self.validity)
@@ -996,10 +1027,11 @@ class Importer(object):
             if not value:
                 return
             if value not in choices_dct.values():
-                raise ValueError(_(u"\"%(value)s\" not in %(values)s") % {
-                    'value':value,
-                    'values':u", ".join([val for val in choices_dct.values()])
+                raise ValueError(
+                    _(u"\"%(value)s\" not in %(values)s") % {
+                        'value': value,
+                        'values': u", ".join([val
+                                              for val in choices_dct.values()])
                     })
             return value
         return function
-