diff options
Diffstat (limited to 'ishtar_common/data_importer.py')
| -rw-r--r-- | ishtar_common/data_importer.py | 74 |
1 files changed, 64 insertions, 10 deletions
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index c2924a541..194a9a5fa 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -88,6 +88,9 @@ class ImporterError(Exception): return self.msg class Formater(object): + def __init__(self, *args, **kwargs): + self.db_target = kwargs.get('db_target', None) + def format(self, value): return value @@ -95,8 +98,10 @@ class Formater(object): return class UnicodeFormater(Formater): - def __init__(self, max_length, clean=False, re_filter=None, notnull=False): + def __init__(self, max_length, clean=False, re_filter=None, notnull=False, + db_target=None): self.max_length = max_length + self.db_target = db_target self.clean = clean self.re_filter = re_filter self.notnull = notnull @@ -182,12 +187,13 @@ class IntegerFormater(Formater): class StrChoiceFormater(Formater): def __init__(self, choices, strict=False, equiv_dict={}, model=None, - cli=False, many_split=''): + cli=False, many_split='', db_target=None): self.choices = list(choices) self.strict = strict self.equiv_dict = copy.deepcopy(equiv_dict) self.cli = cli self.model = model + self.db_target = db_target self.create = False self.missings = set() self.many_split = many_split @@ -200,6 +206,17 @@ class StrChoiceFormater(Formater): if model and v: v = model.objects.get(pk=v) self.equiv_dict[value] = v + if self.db_target: + for target_key in self.db_target.keys.filter(is_set=True).all(): + value = target_key.value + if not self.strict: + value = slugify(value) + if value in self.equiv_dict: + continue + v = target_key.key + if model and v: + v = model.objects.get(pk=v) + self.equiv_dict[value] = v def prepare(self, value): return unicode(value).strip() @@ -257,6 +274,13 @@ class StrChoiceFormater(Formater): unicode(self.equiv_dict[value]))) else: self.equiv_dict[value] = None + if output == 'db' and self.db_target: + for missing in missings: + try: + q = {'target':self.db_target, 'value':missing} + models.TargetKey.objects.create(**q) + except IntegrityError: + pass def new(self, value): return @@ -272,12 +296,14 @@ class StrChoiceFormater(Formater): return self.equiv_dict[value] class TypeFormater(StrChoiceFormater): - def __init__(self, model, cli=False, defaults={}, many_split=False): + def __init__(self, model, cli=False, defaults={}, many_split=False, + db_target=None): self.create = True self.strict = False self.model = model self.defaults = defaults self.many_split = many_split + self.db_target = db_target self.missings = set() self.equiv_dict, self.choices = {}, [] for item in model.objects.all(): @@ -304,8 +330,9 @@ class TypeFormater(StrChoiceFormater): return self.model.objects.create(**values) class DateFormater(Formater): - def __init__(self, date_format="%d/%m/%Y"): + def __init__(self, date_format="%d/%m/%Y", db_target=None): self.date_format = date_format + self.db_target = db_target def format(self, value): value = value.strip() @@ -318,11 +345,26 @@ class DateFormater(Formater): 'value':value}) class StrToBoolean(Formater): - def __init__(self, choices={}, cli=False, strict=False): + def __init__(self, choices={}, cli=False, strict=False, db_target=None): self.dct = copy.copy(choices) self.cli = cli self.strict= strict + self.db_target = db_target self.missings = set() + if self.db_target: + for target_key in self.db_target.keys.filter(is_set=True).all(): + value = target_key.value + value = self.prepare(value) + if value in self.dct: + continue + v = target_key.key + if v in ('False', '0'): + v = False + elif v: + v = True + else: + v = None + self.dct[value] = v def prepare(self, value): value = unicode(value).strip() @@ -359,6 +401,13 @@ class StrToBoolean(Formater): self.dct[value] = False else: self.dct[value] = None + if output == 'db' and self.db_target: + for missing in missings: + try: + q = {'target':self.db_target, 'value':missing} + models.TargetKey.objects.create(**q) + except IntegrityError: + pass def format(self, value): value = self.prepare(value) @@ -388,8 +437,9 @@ class Importer(object): } def __init__(self, skip_lines=0, reference_header=None, - check_col_num=False, test=False, check_validity=True, - history_modifier=None, output='silent'): + check_col_num=False, test=False, check_validity=True, + history_modifier=None, output='silent', + importer_instance=None): """ * skip_line must be set if the data provided has got headers lines. * a reference_header can be provided to perform a data compliance @@ -408,14 +458,18 @@ class Importer(object): self.check_col_num = check_col_num self.check_validity = check_validity self.line_format = copy.copy(self.LINE_FORMAT) + self.importer_instance = importer_instance self._initialized = False self._defaults = self.DEFAULTS.copy() self.history_modifier = history_modifier self.output = output if not self.history_modifier: - # get the first admin - self.history_modifier = User.objects.filter(is_superuser=True - ).order_by('pk')[0] + if self.importer_instance: + self.history_modifier = self.importer_instance.user + else: + # import made by the CLI: get the first admin + self.history_modifier = User.objects.filter( + is_superuser=True).order_by('pk')[0] def initialize(self, table, output='silent'): """ |