diff options
Diffstat (limited to 'ishtar_common/data_importer.py')
| -rw-r--r-- | ishtar_common/data_importer.py | 160 |
1 files changed, 77 insertions, 83 deletions
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index 7ece668c6..7088eee08 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -108,7 +108,7 @@ class ImportFormater(object): def init_db_target(self, user=None): pass - def __unicode__(self): + def __str__(self): return self.field_name def report_succes(self, *args): @@ -214,8 +214,8 @@ class UnicodeFormater(Formater): def format(self, value): try: - if type(value) != unicode: - value = unicode(value.strip()) + if type(value) != str: + value = str(value.strip()) vals = [] for v in value.split(u'\n'): v = v.strip() @@ -285,17 +285,17 @@ class InseeFormater(Formater): try: int(code) except ValueError: - raise ValueError(unicode(self.ERROR).format(value)) + raise ValueError(str(self.ERROR).format(value)) while len(code) < 5: code = "0" + code if len(exp) > 2: - raise ValueError(unicode(self.ERROR).format(value)) + raise ValueError(str(self.ERROR).format(value)) elif len(exp) == 1: return code try: datetime.datetime.strptime(exp[1], '%Y') except ValueError: - raise ValueError(unicode(self.ERROR).format(value)) + raise ValueError(str(self.ERROR).format(value)) return code + u"-" + exp[1] @@ -357,7 +357,7 @@ class StrChoiceFormater(Formater, ChoiceChecker): self.many_split = many_split self.import_instance = None for key, value in self.choices: - value = unicode(value) + value = str(value) if not self.strict: value = slugify(value) if value not in self.equiv_dict: @@ -380,7 +380,7 @@ class StrChoiceFormater(Formater, ChoiceChecker): if key in self.equiv_dict: continue v = target_key.value - if self.model and v and type(v) in (int, unicode): + if self.model and v and type(v) in (int, str): try: v = self.model.objects.get(txt_idx=v) except: @@ -388,21 +388,21 @@ class StrChoiceFormater(Formater, ChoiceChecker): self.equiv_dict[key] = v def prepare(self, value): - return unicode(value).strip() + return str(value).strip() def _get_choices(self, comment=''): msgstr = comment + u" - " - msgstr += unicode(_(u"Choice for \"%s\" is not available. " + msgstr += str(_(u"Choice for \"%s\" is not available. " u"Which one is relevant?\n")) idx = -1 for idx, choice in enumerate(self.choices): msgstr += u"%d. %s\n" % (idx + 1, choice[1]) idx += 2 if self.create: - msgstr += unicode(_(u"%d. None of the above - create new")) % idx \ + msgstr += str(_(u"%d. None of the above - create new")) % idx \ + u"\n" idx += 1 - msgstr += unicode(_(u"%d. None of the above - skip")) % idx + u"\n" + msgstr += str(_(u"%d. None of the above - skip")) % idx + u"\n" return msgstr, idx def check(self, values, output=None, comment='', choose_default=False, @@ -452,7 +452,7 @@ class StrChoiceFormater(Formater, ChoiceChecker): msg = msgstr % value sys.stdout.write(msg.encode('utf-8')) sys.stdout.write("\n>>> ") - res = raw_input() + res = input() try: res = int(res) except ValueError: @@ -468,8 +468,8 @@ class StrChoiceFormater(Formater, ChoiceChecker): elif self.create and res == len(self.choices): self.equiv_dict[value] = self.new(base_value) self.choices.append((self.equiv_dict[value].pk, - unicode(self.equiv_dict[value]))) - self.new_keys[value] = unicode(self.equiv_dict[value]) + str(self.equiv_dict[value]))) + self.new_keys[value] = str(self.equiv_dict[value]) else: self.equiv_dict[value] = None if self.equiv_dict[value] and self.db_target: @@ -540,12 +540,12 @@ class TypeFormater(StrChoiceFormater): self.import_instance = import_instance if self.import_instance: for item in model.objects.all(): - self.choices.append((item.pk, unicode(item))) + self.choices.append((item.pk, str(item))) for key in item.get_keys(importer=import_instance): self.equiv_dict[key] = item def prepare(self, value): - return slugify(unicode(value).strip()) + return slugify(str(value).strip()) def add_key(self, obj, value, importer=None): obj.add_key(slugify(value), force=True, importer=importer) @@ -600,13 +600,14 @@ class FileFormater(Formater): os.mkdir(base_dir) filename = base_dir + os.sep + \ u".".join(items[:-1]) + u'.' + items[-1] + try: - with open(filename, 'w') as f: + with open(filename, 'wb') as f: with zp.open(value) as z: f.write(z.read()) - f = open(filename, 'r') + f = open(filename, 'rb') my_file = File(f) - # manualy set the file size because of an issue with TempFile + # manually set the file size because of an issue with TempFile my_file.size = os.stat(filename).st_size return my_file except KeyError: @@ -639,7 +640,7 @@ class StrToBoolean(Formater, ChoiceChecker): self.dct[key] = v def prepare(self, value): - value = unicode(value).strip() + value = str(value).strip() if not self.strict: value = slugify(value) return value @@ -649,9 +650,9 @@ class StrToBoolean(Formater, ChoiceChecker): if (not output or output == 'silent') and not choose_default: return msgstr = comment + u" - " - msgstr += unicode(_( - u"Choice for \"%s\" is not available. " - u"Which one is relevant?\n")) + msgstr += str(_( + "Choice for \"%s\" is not available. " + "Which one is relevant?\n")) msgstr += u"1. True\n" msgstr += u"2. False\n" msgstr += u"3. Empty\n" @@ -669,7 +670,7 @@ class StrToBoolean(Formater, ChoiceChecker): msg = msgstr % value sys.stdout.write(msg.encode('utf-8')) sys.stdout.write("\n>>> ") - res = raw_input() + res = input() try: res = int(res) except ValueError: @@ -680,7 +681,7 @@ class StrToBoolean(Formater, ChoiceChecker): self.dct[value] = False else: self.dct[value] = None - self.new_keys[value] = unicode(self.dct[value]) + self.new_keys[value] = str(self.dct[value]) if output == 'db' and self.db_target: from ishtar_common.models import TargetKey for missing in self.missings: @@ -811,7 +812,7 @@ class Importer(object): options = formater.model.__module__ + '.' + \ formater.model.__name__ elif formater_name == 'UnicodeFormater': - options = unicode(formater.max_length or '') + options = str(formater.max_length or '') elif formater_name == 'DateFormater': options = formater.date_formats[0] formater_model, created = \ @@ -840,7 +841,7 @@ class Importer(object): if q.count(): cls_name = q.all()[0].name return ImporterError( - unicode(self.ERRORS['improperly_configured']).format(cls_name)) + str(self.ERRORS['improperly_configured']).format(cls_name)) def _get_does_not_exist_in_db_error(self, model, data): from ishtar_common.models import ImporterModel @@ -852,8 +853,7 @@ class Importer(object): [u"{}: {}".format(k, data[k]) for k in data] ) raise ImporterError( - unicode(self.ERRORS['does_not_exist_in_db'] - ).format(cls_name, values)) + str(self.ERRORS['does_not_exist_in_db']).format(cls_name, values)) def __init__(self, skip_lines=0, reference_header=None, check_col_num=False, test=False, history_modifier=None, @@ -1033,15 +1033,15 @@ class Importer(object): elif force_value and value: if concat_str and key in current_data \ and current_data[key]: - current_data[key] = unicode(current_data[key]) + \ - concat_str + unicode(value) + current_data[key] = str(current_data[key]) + \ + concat_str + str(value) else: current_data[key] = value elif key not in current_data or not current_data[key]: current_data[key] = value elif concat_str: - current_data[key] = unicode(current_data[key]) +\ - concat_str + unicode(value) + current_data[key] = str(current_data[key]) +\ + concat_str + str(value) if force_new: current_data['__force_new'] = True elif key not in current_data: @@ -1102,7 +1102,7 @@ class Importer(object): sys.stdout.flush() try: results.append(self._line_processing(idx_line, line)) - except ImporterError, msg: + except ImporterError as msg: self.errors.append((idx_line, None, msg)) return results @@ -1130,7 +1130,7 @@ class Importer(object): self.current_csv_line = line n = datetime.datetime.now() - logger.debug('%s - Processing line %d' % (unicode(n - self.now), + logger.debug('%s - Processing line %d' % (str(n - self.now), idx_line)) self.now = n n2 = n @@ -1151,7 +1151,7 @@ class Importer(object): if self.c_errors: return n = datetime.datetime.now() - logger.debug('* %s - Cols read' % (unicode(n - n2))) + logger.debug('* %s - Cols read' % (str(n - n2))) n2 = n if self.test: return @@ -1183,11 +1183,11 @@ class Importer(object): setattr(obj, k, data['defaults'][k]) obj.save() n = datetime.datetime.now() - logger.debug('* %s - Item saved' % (unicode(n - n2))) + logger.debug('* %s - Item saved' % (str(n - n2))) n2 = n for formater, value in self._throughs: n = datetime.datetime.now() - logger.debug('* %s - Processing formater %s' % (unicode(n - n2), + logger.debug('* %s - Processing formater %s' % (str(n - n2), formater.field_name)) n2 = n data = {} @@ -1199,7 +1199,7 @@ class Importer(object): through_cls = formater.through if formater.through_unicity_keys: data['defaults'] = {} - for k in data.keys(): + for k in list(data.keys()): if k not in formater.through_unicity_keys \ and k != 'defaults': data['defaults'][k] = data.pop(k) @@ -1270,7 +1270,7 @@ class Importer(object): val = val.replace(NEW_LINE_BREAK, '\n') self.errors.append( (idx_line + 1, idx_col + 1, - unicode(self.ERRORS['regex_not_match']) + val)) + str(self.ERRORS['regex_not_match']) + val)) c_row.append("") return val_group = [] @@ -1288,7 +1288,7 @@ class Importer(object): func = formater.formater if type(func) in (list, tuple): func = func[idx_v] - if not callable(func) and type(func) in (unicode, str): + if not callable(func) and type(func) == str: func = getattr(self, func) values = [v] @@ -1334,10 +1334,10 @@ class Importer(object): value = func.format(v, archive=self.archive) else: value = func.format(v) - except ValueError, e: + except ValueError as e: if formater.required: self.c_errors = True - self.errors.append((idx_line + 1, idx_col + 1, e.message)) + self.errors.append((idx_line + 1, idx_col + 1, str(e))) c_values.append('') return if formater.value_format and value is not None and value != "": @@ -1357,10 +1357,9 @@ class Importer(object): printed_values = [value] try: # don't reunicode - unicoded values - c_values.append(u" ; ".join([v for v in printed_values])) + c_values.append(" ; ".join([v for v in printed_values])) except TypeError: - c_values.append(u" ; ".join([unicode(v) - for v in printed_values])) + c_values.append(" ; ".join([str(v) for v in printed_values])) if value is None and formater.required: self.c_errors = True self.errors.append((idx_line + 1, idx_col + 1, @@ -1449,7 +1448,7 @@ class Importer(object): vals.append(default_dict.copy()) # # manage multiple values - for key in val.keys(): + for key in list(val.keys()): if type(val[key]) in (list, tuple): for idx, v in enumerate(val[key]): if len(vals) <= idx: @@ -1475,7 +1474,7 @@ class Importer(object): self.history_modifier m2m_m2ms = [] c_c_path = c_path[:] - for k in v.keys(): + for k in list(v.keys()): if k not in field_names: continue self.get_field(model, k, v, m2m_m2ms, c_c_path, @@ -1518,15 +1517,14 @@ class Importer(object): **v) except FieldError as e: raise ImporterError( - unicode( - _(u"Importer configuration error: " - u"\"{}\".")).format(e.message)) + str( + _("Importer configuration error: " + "\"{}\".")).format(e)) except Exception as e: - msg = unicode( - _(u"Import error: {} - \"{}\".") - ).format(model, e.message.decode('utf-8')) - e.message = msg - raise e + msg = str( + _("Import error: {} - \"{}\".") + ).format(model, e) + raise e(msg) else: get_v = v.copy() if 'defaults' in get_v: @@ -1603,16 +1601,15 @@ class Importer(object): try: field_object = cls._meta.get_field(attribute) except FieldDoesNotExist: - raise ImporterError(unicode( - _(u"Importer configuration error: field \"{}\" does not exist " - u"for {}.")).format(attribute, cls._meta.verbose_name)) + raise ImporterError(str( + _("Importer configuration error: field \"{}\" does not exist " + "for {}.")).format(attribute, cls._meta.verbose_name)) if field_object.many_to_many: try: m2ms += self._get_field_m2m(attribute, data, c_path, new_created, field_object) except Exception as e: - self.errors.append( - (self.idx_line, None, e.message.decode('utf-8'))) + self.errors.append((self.idx_line, None, str(e))) return if not hasattr(field_object, 'rel') or not field_object.rel: return @@ -1680,17 +1677,17 @@ class Importer(object): new_created) except (ValueError, IntegrityError, FieldDoesNotExist) as e: try: - message = e.message.decode('utf-8') + message = str(e) except (UnicodeDecodeError, UnicodeDecodeError): message = '' try: - data = unicode(data) + data = str(data) except UnicodeDecodeError: data = '' raise ImporterError( "Erreur d'import %s %s, contexte : %s, erreur : %s" - % (unicode(cls), unicode("__".join(path)), - unicode(data), message)) + % (str(cls), str("__".join(path)), + str(data), message)) create_dict = copy.deepcopy(data) for k in create_dict.keys(): @@ -1740,7 +1737,7 @@ class Importer(object): else: # manage UNICITY_KEYS - only level 1 if not path and self.UNICITY_KEYS: - for k in dct.keys(): + for k in list(dct.keys()): if k not in self.UNICITY_KEYS \ and k != 'defaults': if dct[k]: @@ -1774,8 +1771,8 @@ class Importer(object): return _(u"* the query match more than 10 " u"results*"), False else: - return unicode(_(u" or ")).join( - [unicode(item) for item in q.all()] + return str(_(u" or ")).join( + [str(item) for item in q.all()] ), False else: self.updated_objects.append( @@ -1805,8 +1802,8 @@ class Importer(object): if val is None or val == '': updated_dct[k] = new_val elif k in self.concats \ - and type(val) == unicode \ - and type(new_val) == unicode: + and type(val) == str \ + and type(new_val) == str: updated_dct[k] = val + u"\n" + new_val else: for k in dct['defaults']: @@ -1830,12 +1827,12 @@ class Importer(object): obj.imports.add(self.import_instance) except (ValueError, IntegrityError, DatabaseError, GEOSException) as e: - raise IntegrityError(e.message) + raise IntegrityError(str(e)) except cls.MultipleObjectsReturned as e: created = False if 'defaults' in dct: dct.pop('defaults') - raise IntegrityError(e.message) + raise IntegrityError(str(e)) # obj = cls.objects.filter(**dct).all()[0] for attr, value in m2ms: values = [value] @@ -1905,26 +1902,23 @@ class Importer(object): obj.fix() except IntegrityError as e: try: - message = e.message.decode('utf-8') + message = str(e) except (UnicodeDecodeError, UnicodeDecodeError): message = '' try: - data = unicode(data) + data = str(data) except UnicodeDecodeError: data = '' raise ImporterError( u"Erreur d'import %s %s, contexte : %s, erreur : %s" - % (unicode(cls), unicode("__".join(path)), - unicode(data), message)) + % (str(cls), str("__".join(path)), + str(data), message)) return obj, created def _format_csv_line(self, values, empty=u"-"): - try: - return u'"' + u'","'.join( - [(v and unicode(v).replace('"', '""')) or empty - for v in values]) + u'"' - except UnicodeDecodeError: - return "" + return u'"' + u'","'.join( + [(v and str(v).replace('"', '""')) or empty + for v in values]) + u'"' def _get_csv(self, rows, header=[], empty=u"-"): if not rows: |