Migrate to python 3 - Clean old migrations and some old scripts

1 files changed, 77 insertions, 83 deletions

diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index 7ece668c6..7088eee08 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -108,7 +108,7 @@ class ImportFormater(object):
     def init_db_target(self, user=None):
         pass
 
-    def __unicode__(self):
+    def __str__(self):
         return self.field_name
 
     def report_succes(self, *args):
@@ -214,8 +214,8 @@ class UnicodeFormater(Formater):
 
     def format(self, value):
         try:
-            if type(value) != unicode:
-                value = unicode(value.strip())
+            if type(value) != str:
+                value = str(value.strip())
             vals = []
             for v in value.split(u'\n'):
                 v = v.strip()
@@ -285,17 +285,17 @@ class InseeFormater(Formater):
         try:
             int(code)
         except ValueError:
-            raise ValueError(unicode(self.ERROR).format(value))
+            raise ValueError(str(self.ERROR).format(value))
         while len(code) < 5:
             code = "0" + code
         if len(exp) > 2:
-            raise ValueError(unicode(self.ERROR).format(value))
+            raise ValueError(str(self.ERROR).format(value))
         elif len(exp) == 1:
             return code
         try:
             datetime.datetime.strptime(exp[1], '%Y')
         except ValueError:
-            raise ValueError(unicode(self.ERROR).format(value))
+            raise ValueError(str(self.ERROR).format(value))
         return code + u"-" + exp[1]
 
 
@@ -357,7 +357,7 @@ class StrChoiceFormater(Formater, ChoiceChecker):
         self.many_split = many_split
         self.import_instance = None
         for key, value in self.choices:
-            value = unicode(value)
+            value = str(value)
             if not self.strict:
                 value = slugify(value)
             if value not in self.equiv_dict:
@@ -380,7 +380,7 @@ class StrChoiceFormater(Formater, ChoiceChecker):
             if key in self.equiv_dict:
                 continue
             v = target_key.value
-            if self.model and v and type(v) in (int, unicode):
+            if self.model and v and type(v) in (int, str):
                 try:
                     v = self.model.objects.get(txt_idx=v)
                 except:
@@ -388,21 +388,21 @@ class StrChoiceFormater(Formater, ChoiceChecker):
             self.equiv_dict[key] = v
 
     def prepare(self, value):
-        return unicode(value).strip()
+        return str(value).strip()
 
     def _get_choices(self, comment=''):
         msgstr = comment + u" - "
-        msgstr += unicode(_(u"Choice for \"%s\" is not available. "
+        msgstr += str(_(u"Choice for \"%s\" is not available. "
                             u"Which one is relevant?\n"))
         idx = -1
         for idx, choice in enumerate(self.choices):
             msgstr += u"%d. %s\n" % (idx + 1, choice[1])
         idx += 2
         if self.create:
-            msgstr += unicode(_(u"%d. None of the above - create new")) % idx \
+            msgstr += str(_(u"%d. None of the above - create new")) % idx \
                 + u"\n"
             idx += 1
-        msgstr += unicode(_(u"%d. None of the above - skip")) % idx + u"\n"
+        msgstr += str(_(u"%d. None of the above - skip")) % idx + u"\n"
         return msgstr, idx
 
     def check(self, values, output=None, comment='', choose_default=False,
@@ -452,7 +452,7 @@ class StrChoiceFormater(Formater, ChoiceChecker):
                 msg = msgstr % value
                 sys.stdout.write(msg.encode('utf-8'))
                 sys.stdout.write("\n>>> ")
-                res = raw_input()
+                res = input()
                 try:
                     res = int(res)
                 except ValueError:
@@ -468,8 +468,8 @@ class StrChoiceFormater(Formater, ChoiceChecker):
             elif self.create and res == len(self.choices):
                 self.equiv_dict[value] = self.new(base_value)
                 self.choices.append((self.equiv_dict[value].pk,
-                                     unicode(self.equiv_dict[value])))
-                self.new_keys[value] = unicode(self.equiv_dict[value])
+                                     str(self.equiv_dict[value])))
+                self.new_keys[value] = str(self.equiv_dict[value])
             else:
                 self.equiv_dict[value] = None
             if self.equiv_dict[value] and self.db_target:
@@ -540,12 +540,12 @@ class TypeFormater(StrChoiceFormater):
         self.import_instance = import_instance
         if self.import_instance:
             for item in model.objects.all():
-                self.choices.append((item.pk, unicode(item)))
+                self.choices.append((item.pk, str(item)))
                 for key in item.get_keys(importer=import_instance):
                     self.equiv_dict[key] = item
 
     def prepare(self, value):
-        return slugify(unicode(value).strip())
+        return slugify(str(value).strip())
 
     def add_key(self, obj, value, importer=None):
         obj.add_key(slugify(value), force=True, importer=importer)
@@ -600,13 +600,14 @@ class FileFormater(Formater):
             os.mkdir(base_dir)
         filename = base_dir + os.sep + \
             u".".join(items[:-1]) + u'.' + items[-1]
+
         try:
-            with open(filename, 'w') as f:
+            with open(filename, 'wb') as f:
                 with zp.open(value) as z:
                     f.write(z.read())
-            f = open(filename, 'r')
+            f = open(filename, 'rb')
             my_file = File(f)
-            # manualy set the file size because of an issue with TempFile
+            # manually set the file size because of an issue with TempFile
             my_file.size = os.stat(filename).st_size
             return my_file
         except KeyError:
@@ -639,7 +640,7 @@ class StrToBoolean(Formater, ChoiceChecker):
             self.dct[key] = v
 
     def prepare(self, value):
-        value = unicode(value).strip()
+        value = str(value).strip()
         if not self.strict:
             value = slugify(value)
         return value
@@ -649,9 +650,9 @@ class StrToBoolean(Formater, ChoiceChecker):
         if (not output or output == 'silent') and not choose_default:
             return
         msgstr = comment + u" - "
-        msgstr += unicode(_(
-            u"Choice for \"%s\" is not available. "
-            u"Which one is relevant?\n"))
+        msgstr += str(_(
+            "Choice for \"%s\" is not available. "
+            "Which one is relevant?\n"))
         msgstr += u"1. True\n"
         msgstr += u"2. False\n"
         msgstr += u"3. Empty\n"
@@ -669,7 +670,7 @@ class StrToBoolean(Formater, ChoiceChecker):
                 msg = msgstr % value
                 sys.stdout.write(msg.encode('utf-8'))
                 sys.stdout.write("\n>>> ")
-                res = raw_input()
+                res = input()
                 try:
                     res = int(res)
                 except ValueError:
@@ -680,7 +681,7 @@ class StrToBoolean(Formater, ChoiceChecker):
                 self.dct[value] = False
             else:
                 self.dct[value] = None
-            self.new_keys[value] = unicode(self.dct[value])
+            self.new_keys[value] = str(self.dct[value])
         if output == 'db' and self.db_target:
             from ishtar_common.models import TargetKey
             for missing in self.missings:
@@ -811,7 +812,7 @@ class Importer(object):
                     options = formater.model.__module__ + '.' + \
                         formater.model.__name__
                 elif formater_name == 'UnicodeFormater':
-                    options = unicode(formater.max_length or '')
+                    options = str(formater.max_length or '')
                 elif formater_name == 'DateFormater':
                     options = formater.date_formats[0]
                 formater_model, created = \
@@ -840,7 +841,7 @@ class Importer(object):
         if q.count():
             cls_name = q.all()[0].name
         return ImporterError(
-            unicode(self.ERRORS['improperly_configured']).format(cls_name))
+            str(self.ERRORS['improperly_configured']).format(cls_name))
 
     def _get_does_not_exist_in_db_error(self, model, data):
         from ishtar_common.models import ImporterModel
@@ -852,8 +853,7 @@ class Importer(object):
             [u"{}: {}".format(k, data[k]) for k in data]
         )
         raise ImporterError(
-            unicode(self.ERRORS['does_not_exist_in_db']
-                    ).format(cls_name, values))
+            str(self.ERRORS['does_not_exist_in_db']).format(cls_name, values))
 
     def __init__(self, skip_lines=0, reference_header=None,
                  check_col_num=False, test=False, history_modifier=None,
@@ -1033,15 +1033,15 @@ class Importer(object):
                     elif force_value and value:
                         if concat_str and key in current_data \
                                 and current_data[key]:
-                            current_data[key] = unicode(current_data[key]) + \
-                                concat_str + unicode(value)
+                            current_data[key] = str(current_data[key]) + \
+                                concat_str + str(value)
                         else:
                             current_data[key] = value
                     elif key not in current_data or not current_data[key]:
                         current_data[key] = value
                     elif concat_str:
-                        current_data[key] = unicode(current_data[key]) +\
-                            concat_str + unicode(value)
+                        current_data[key] = str(current_data[key]) +\
+                            concat_str + str(value)
                     if force_new:
                         current_data['__force_new'] = True
                 elif key not in current_data:
@@ -1102,7 +1102,7 @@ class Importer(object):
                 sys.stdout.flush()
             try:
                 results.append(self._line_processing(idx_line, line))
-            except ImporterError, msg:
+            except ImporterError as msg:
                 self.errors.append((idx_line, None, msg))
         return results
 
@@ -1130,7 +1130,7 @@ class Importer(object):
             self.current_csv_line = line
 
         n = datetime.datetime.now()
-        logger.debug('%s - Processing line %d' % (unicode(n - self.now),
+        logger.debug('%s - Processing line %d' % (str(n - self.now),
                                                   idx_line))
         self.now = n
         n2 = n
@@ -1151,7 +1151,7 @@ class Importer(object):
         if self.c_errors:
             return
         n = datetime.datetime.now()
-        logger.debug('* %s - Cols read' % (unicode(n - n2)))
+        logger.debug('* %s - Cols read' % (str(n - n2)))
         n2 = n
         if self.test:
             return
@@ -1183,11 +1183,11 @@ class Importer(object):
                 setattr(obj, k, data['defaults'][k])
             obj.save()
         n = datetime.datetime.now()
-        logger.debug('* %s - Item saved' % (unicode(n - n2)))
+        logger.debug('* %s - Item saved' % (str(n - n2)))
         n2 = n
         for formater, value in self._throughs:
             n = datetime.datetime.now()
-            logger.debug('* %s - Processing formater %s' % (unicode(n - n2),
+            logger.debug('* %s - Processing formater %s' % (str(n - n2),
                          formater.field_name))
             n2 = n
             data = {}
@@ -1199,7 +1199,7 @@ class Importer(object):
             through_cls = formater.through
             if formater.through_unicity_keys:
                 data['defaults'] = {}
-                for k in data.keys():
+                for k in list(data.keys()):
                     if k not in formater.through_unicity_keys \
                        and k != 'defaults':
                         data['defaults'][k] = data.pop(k)
@@ -1270,7 +1270,7 @@ class Importer(object):
                 val = val.replace(NEW_LINE_BREAK, '\n')
                 self.errors.append(
                     (idx_line + 1, idx_col + 1,
-                     unicode(self.ERRORS['regex_not_match']) + val))
+                     str(self.ERRORS['regex_not_match']) + val))
                 c_row.append("")
                 return
             val_group = []
@@ -1288,7 +1288,7 @@ class Importer(object):
             func = formater.formater
             if type(func) in (list, tuple):
                 func = func[idx_v]
-            if not callable(func) and type(func) in (unicode, str):
+            if not callable(func) and type(func) == str:
                 func = getattr(self, func)
 
             values = [v]
@@ -1334,10 +1334,10 @@ class Importer(object):
                             value = func.format(v, archive=self.archive)
                         else:
                             value = func.format(v)
-                except ValueError, e:
+                except ValueError as e:
                     if formater.required:
                         self.c_errors = True
-                    self.errors.append((idx_line + 1, idx_col + 1, e.message))
+                    self.errors.append((idx_line + 1, idx_col + 1, str(e)))
                     c_values.append('')
                     return
                 if formater.value_format and value is not None and value != "":
@@ -1357,10 +1357,9 @@ class Importer(object):
                 printed_values = [value]
             try:
                 # don't reunicode - unicoded values
-                c_values.append(u" ; ".join([v for v in printed_values]))
+                c_values.append(" ; ".join([v for v in printed_values]))
             except TypeError:
-                c_values.append(u" ; ".join([unicode(v)
-                                             for v in printed_values]))
+                c_values.append(" ; ".join([str(v) for v in printed_values]))
             if value is None and formater.required:
                 self.c_errors = True
                 self.errors.append((idx_line + 1, idx_col + 1,
@@ -1449,7 +1448,7 @@ class Importer(object):
             vals.append(default_dict.copy())
 
             # # manage multiple values
-            for key in val.keys():
+            for key in list(val.keys()):
                 if type(val[key]) in (list, tuple):
                     for idx, v in enumerate(val[key]):
                         if len(vals) <= idx:
@@ -1475,7 +1474,7 @@ class Importer(object):
                         self.history_modifier
                 m2m_m2ms = []
                 c_c_path = c_path[:]
-                for k in v.keys():
+                for k in list(v.keys()):
                     if k not in field_names:
                         continue
                     self.get_field(model, k, v, m2m_m2ms, c_c_path,
@@ -1518,15 +1517,14 @@ class Importer(object):
                                 **v)
                         except FieldError as e:
                             raise ImporterError(
-                                unicode(
-                                    _(u"Importer configuration error: "
-                                      u"\"{}\".")).format(e.message))
+                                str(
+                                    _("Importer configuration error: "
+                                      "\"{}\".")).format(e))
                         except Exception as e:
-                            msg = unicode(
-                                _(u"Import error: {} - \"{}\".")
-                            ).format(model, e.message.decode('utf-8'))
-                            e.message = msg
-                            raise e
+                            msg = str(
+                                _("Import error: {} - \"{}\".")
+                            ).format(model, e)
+                            raise e(msg)
                     else:
                         get_v = v.copy()
                         if 'defaults' in get_v:
@@ -1603,16 +1601,15 @@ class Importer(object):
         try:
             field_object = cls._meta.get_field(attribute)
         except FieldDoesNotExist:
-            raise ImporterError(unicode(
-                _(u"Importer configuration error: field \"{}\" does not exist "
-                  u"for {}.")).format(attribute, cls._meta.verbose_name))
+            raise ImporterError(str(
+                _("Importer configuration error: field \"{}\" does not exist "
+                  "for {}.")).format(attribute, cls._meta.verbose_name))
         if field_object.many_to_many:
             try:
                 m2ms += self._get_field_m2m(attribute, data, c_path,
                                             new_created, field_object)
             except Exception as e:
-                self.errors.append(
-                    (self.idx_line, None, e.message.decode('utf-8')))
+                self.errors.append((self.idx_line, None, str(e)))
             return
         if not hasattr(field_object, 'rel') or not field_object.rel:
             return
@@ -1680,17 +1677,17 @@ class Importer(object):
                                    new_created)
         except (ValueError, IntegrityError, FieldDoesNotExist) as e:
             try:
-                message = e.message.decode('utf-8')
+                message = str(e)
             except (UnicodeDecodeError, UnicodeDecodeError):
                 message = ''
             try:
-                data = unicode(data)
+                data = str(data)
             except UnicodeDecodeError:
                 data = ''
             raise ImporterError(
                 "Erreur d'import %s %s, contexte : %s, erreur : %s"
-                % (unicode(cls), unicode("__".join(path)),
-                   unicode(data), message))
+                % (str(cls), str("__".join(path)),
+                   str(data), message))
 
         create_dict = copy.deepcopy(data)
         for k in create_dict.keys():
@@ -1740,7 +1737,7 @@ class Importer(object):
                 else:
                     # manage UNICITY_KEYS - only level 1
                     if not path and self.UNICITY_KEYS:
-                        for k in dct.keys():
+                        for k in list(dct.keys()):
                             if k not in self.UNICITY_KEYS \
                                and k != 'defaults':
                                 if dct[k]:
@@ -1774,8 +1771,8 @@ class Importer(object):
                                 return _(u"* the query match more than 10 "
                                          u"results*"), False
                             else:
-                                return unicode(_(u" or ")).join(
-                                    [unicode(item) for item in q.all()]
+                                return str(_(u" or ")).join(
+                                    [str(item) for item in q.all()]
                                 ), False
                         else:
                             self.updated_objects.append(
@@ -1805,8 +1802,8 @@ class Importer(object):
                                 if val is None or val == '':
                                     updated_dct[k] = new_val
                                 elif k in self.concats \
-                                        and type(val) == unicode \
-                                        and type(new_val) == unicode:
+                                        and type(val) == str \
+                                        and type(new_val) == str:
                                     updated_dct[k] = val + u"\n" + new_val
                         else:
                             for k in dct['defaults']:
@@ -1830,12 +1827,12 @@ class Importer(object):
                     obj.imports.add(self.import_instance)
             except (ValueError, IntegrityError, DatabaseError,
                     GEOSException) as e:
-                raise IntegrityError(e.message)
+                raise IntegrityError(str(e))
             except cls.MultipleObjectsReturned as e:
                 created = False
                 if 'defaults' in dct:
                     dct.pop('defaults')
-                raise IntegrityError(e.message)
+                raise IntegrityError(str(e))
                 # obj = cls.objects.filter(**dct).all()[0]
             for attr, value in m2ms:
                 values = [value]
@@ -1905,26 +1902,23 @@ class Importer(object):
                 obj.fix()
         except IntegrityError as e:
             try:
-                message = e.message.decode('utf-8')
+                message = str(e)
             except (UnicodeDecodeError, UnicodeDecodeError):
                 message = ''
             try:
-                data = unicode(data)
+                data = str(data)
             except UnicodeDecodeError:
                 data = ''
             raise ImporterError(
                 u"Erreur d'import %s %s, contexte : %s, erreur : %s"
-                % (unicode(cls), unicode("__".join(path)),
-                   unicode(data), message))
+                % (str(cls), str("__".join(path)),
+                   str(data), message))
         return obj, created
 
     def _format_csv_line(self, values, empty=u"-"):
-        try:
-            return u'"' + u'","'.join(
-                [(v and unicode(v).replace('"', '""')) or empty
-                 for v in values]) + u'"'
-        except UnicodeDecodeError:
-            return ""
+        return u'"' + u'","'.join(
+            [(v and str(v).replace('"', '""')) or empty
+             for v in values]) + u'"'
 
     def _get_csv(self, rows, header=[], empty=u"-"):
         if not rows: