diff options
Diffstat (limited to 'ishtar_common/data_importer.py')
| -rw-r--r-- | ishtar_common/data_importer.py | 85 |
1 files changed, 67 insertions, 18 deletions
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index 2963fa680..3db03d916 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -70,7 +70,7 @@ class ImportFormater(object): lst = [self.formater] for formater in lst: if formater: - formater.check(vals, output) + formater.check(vals, output, self.comment) def post_process(self, obj, context, value, owner=None): raise NotImplemented() @@ -92,9 +92,19 @@ class Formater(object): def format(self, value): return value - def check(self, values, output=None): + def check(self, values, output=None, comment=''): return +class ChoiceChecker(object): + def report_new(self, comment): + if not self.new_keys: + return + msg = u"For \"%s\" these new associations have been made:\n" % comment + sys.stderr.write(msg.encode('utf-8')) + for k in self.new_keys: + msg = u'"%s";"%s"\n' % (k, self.new_keys[k]) + sys.stderr.write(msg.encode('utf-8')) + class UnicodeFormater(Formater): def __init__(self, max_length, clean=False, re_filter=None, notnull=False, db_target=None): @@ -183,7 +193,7 @@ class IntegerFormater(Formater): raise ValueError(_(u"\"%(value)s\" is not an integer") % { 'value':value}) -class StrChoiceFormater(Formater): +class StrChoiceFormater(Formater, ChoiceChecker): def __init__(self, choices, strict=False, equiv_dict={}, model=None, cli=False, many_split='', db_target=None): self.choices = list(choices) @@ -194,6 +204,7 @@ class StrChoiceFormater(Formater): self.db_target = db_target self.create = False self.missings = set() + self.new_keys = {} self.many_split = many_split for key, value in self.choices: value = unicode(value) @@ -219,8 +230,9 @@ class StrChoiceFormater(Formater): def prepare(self, value): return unicode(value).strip() - def _get_choices(self): - msgstr = unicode(_(u"Choice for \"%s\" is not available. "\ + def _get_choices(self, comment=''): + msgstr = comment + u" - " + msgstr += unicode(_(u"Choice for \"%s\" is not available. "\ u"Which one is relevant?\n")) idx = -1 for idx, choice in enumerate(self.choices): @@ -233,7 +245,7 @@ class StrChoiceFormater(Formater): msgstr += unicode(_(u"%d. None of the above - skip")) % idx + u"\n" return msgstr, idx - def check(self, values, output=None): + def check(self, values, output=None, comment=''): if not output or output == 'silent': return if self.many_split: @@ -250,11 +262,13 @@ class StrChoiceFormater(Formater): if output != 'cli': self.missings.add(value) continue - msgstr, idx = self._get_choices() + msgstr, idx = self._get_choices(comment) res = None while res not in range(1, idx+1): - sys.stdout.write(msgstr % value) - res = raw_input(">>> ") + msg = msgstr % value + sys.stdout.write(msg.encode('utf-8')) + sys.stdout.write("\n>>> ") + res = raw_input() try: res = int(res) except ValueError: @@ -266,10 +280,12 @@ class StrChoiceFormater(Formater): v = self.model.objects.get(pk=v) self.equiv_dict[value] = v self.add_key(v, value) + self.new_keys[value] = v elif self.create and res == len(self.choices): self.equiv_dict[value] = self.new(base_value) self.choices.append((self.equiv_dict[value].pk, unicode(self.equiv_dict[value]))) + self.new_keys[value] = unicode(self.equiv_dict[value]) else: self.equiv_dict[value] = None if output == 'db' and self.db_target: @@ -283,6 +299,8 @@ class StrChoiceFormater(Formater): TargetKey.objects.create(**q) except IntegrityError: pass + if output == 'cli': + self.report_new(comment) def new(self, value): return @@ -346,7 +364,7 @@ class DateFormater(Formater): raise ValueError(_(u"\"%(value)s\" is not a valid date") % { 'value':value}) -class StrToBoolean(Formater): +class StrToBoolean(Formater, ChoiceChecker): def __init__(self, choices={}, cli=False, strict=False, db_target=None): self.dct = copy.copy(choices) self.cli = cli @@ -367,6 +385,7 @@ class StrToBoolean(Formater): else: v = None self.dct[value] = v + self.new_keys = {} def prepare(self, value): value = unicode(value).strip() @@ -374,10 +393,11 @@ class StrToBoolean(Formater): value = slugify(value) return value - def check(self, values, output=None): + def check(self, values, output=None, comment=''): if not output or output == 'silent': return - msgstr = unicode(_(u"Choice for \"%s\" is not available. "\ + msgstr = comment + u" - " + msgstr += unicode(_(u"Choice for \"%s\" is not available. "\ u"Which one is relevant?\n")) msgstr += u"1. True\n" msgstr += u"2. False\n" @@ -391,8 +411,10 @@ class StrToBoolean(Formater): continue res = None while res not in range(1, 4): - sys.stdout.write(msgstr % value) - res = raw_input(">>> ") + msg = msgstr % value + sys.stdout.write(msg.encode('utf-8')) + sys.stdout.write("\n>>> ") + res = raw_input() try: res = int(res) except ValueError: @@ -411,6 +433,9 @@ class StrToBoolean(Formater): models.TargetKey.objects.create(**q) except IntegrityError: pass + self.new_keys[value] = unicode(self.dct[value]) + if output == 'cli': + self.report_new(comment) def format(self, value): value = self.prepare(value) @@ -461,6 +486,8 @@ class Importer(object): self._defaults = self.DEFAULTS.copy() self.history_modifier = history_modifier self.output = output + self.result = [] + self.result_cols = [] if not self.history_modifier: if self.import_instance: self.history_modifier = self.import_instance.user @@ -560,16 +587,25 @@ class Importer(object): time_by_item = ellapsed/idx_line if time_by_item: left = ((total - idx_line)*time_by_item).seconds - txt = "\r* %d/%d" % (idx_line+1, total) + txt = u"\r* %d/%d" % (idx_line+1, total) if left: - txt += " (%d seconds left)" % left - sys.stdout.write(txt) + txt += u" (%d seconds left)" % left + sys.stdout.write(txt.encode('utf-8')) sys.stdout.flush() try: self._line_processing(idx_line, line) except ImporterError, msg: self.errors.append((idx_line, None, msg)) + report_name = 'report-%s.csv' % datetime.datetime.now().isoformat() + with open(report_name, 'w') as validity_file: + self.validity_file = UnicodeWriter(validity_file, + delimiter=',', quotechar='"', + quoting=csv.QUOTE_MINIMAL) + self.validity_file.writerow(self.result_cols) + for line in self.result: + self.validity_file.writerow(line) + def _line_processing(self, idx_line, line): if self.skip_lines > idx_line: self.validity.append(line) @@ -669,6 +705,19 @@ class Importer(object): for formater, val in self._post_processing: formater.post_process(obj, data, val, owner=self.history_modifier) + # writing report + self.result.append([]) + for k in data.keys(): + if k not in self.result_cols: + self.result_cols.append(k) + for k in self.result_cols: + if hasattr(obj, k): + self.result[-1].append(unicode(getattr(obj, k))) + elif k not in data or not data[k]: + self.result[-1].append('') + else: + self.result[-1].append(unicode(data[k])) + def _row_processing(self, c_row, idx_col, idx_line, val, data): if idx_col >= len(self.line_format): return @@ -827,7 +876,7 @@ class Importer(object): obj.imports.add(self.import_instance) except IntegrityError as e: raise IntegrityError(e.message) - except: + except cls.MultipleObjectsReturned: created = False obj = cls.objects.filter(**create_dict).all()[0] for attr, value in m2ms: |