diff options
Diffstat (limited to 'ishtar_common/models_imports.py')
| -rw-r--r-- | ishtar_common/models_imports.py | 910 |
1 files changed, 910 insertions, 0 deletions
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py new file mode 100644 index 000000000..d638c76f1 --- /dev/null +++ b/ishtar_common/models_imports.py @@ -0,0 +1,910 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (C) 2017 Étienne Loks <etienne.loks_AT_peacefrogsDOTnet> + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# See the file COPYING for details. + +import csv +import datetime +from importlib import import_module +import os +import logging +import shutil +import re +import tempfile +import unicodecsv +import zipfile + +from django.conf import settings +from django.contrib.gis.db import models +from django.core.exceptions import SuspiciousOperation +from django.core.files.base import ContentFile +from django.db.models.base import ModelBase +from django.db.models.signals import pre_delete +from django.template.defaultfilters import slugify +from django.utils.translation import ugettext_lazy as _, pgettext_lazy + +from ishtar_common.utils import create_slug, \ + get_all_related_m2m_objects_with_model +from ishtar_common.data_importer import Importer, ImportFormater, \ + IntegerFormater, FloatFormater, UnicodeFormater, DateFormater, \ + TypeFormater, YearFormater, StrToBoolean, FileFormater + +logger = logging.getLogger(__name__) + +IMPORTER_CLASSES = {} + +IMPORTER_CLASSES.update({ + 'sra-pdl-files': + 'archaeological_files.data_importer.FileImporterSraPdL'}) + + +def get_model_fields(model): + """ + Return a dict of fields from model + To be replace in Django 1.8 with get_fields, get_field + """ + fields = {} + options = model._meta + for field in sorted(options.fields + options.many_to_many): + fields[field.name] = field + if hasattr(model, 'get_extra_fields'): + fields.update(model.get_extra_fields()) + return fields + + +def import_class(full_path_classname): + """ + Return the model class from the full path + TODO: add a white list for more security + """ + mods = full_path_classname.split('.') + if len(mods) == 1: + mods = ['ishtar_common', 'models', mods[0]] + elif 'models' not in mods and 'models_finds' not in mods \ + and 'models_treatments' not in mods: + raise SuspiciousOperation( + u"Try to import a non model from a string") + module = import_module('.'.join(mods[:-1])) + return getattr(module, mods[-1]) + + +class ImportModelManager(models.Manager): + def get_by_natural_key(self, klass): + return self.get(klass=klass) + + +class ImporterModel(models.Model): + name = models.CharField(_(u"Name"), max_length=200) + klass = models.CharField(_(u"Class name"), max_length=200, unique=True) + objects = ImportModelManager() + + class Meta: + verbose_name = _(u"Importer - Model") + verbose_name_plural = _(u"Importer - Models") + ordering = ('name',) + + def __unicode__(self): + return self.name + + def natural_key(self): + return (self.klass, ) + + +class ImporterTypeManager(models.Manager): + def get_by_natural_key(self, slug): + return self.get(slug=slug) + + +class ImporterType(models.Model): + """ + Description of a table to be mapped with ishtar database + """ + name = models.CharField(_(u"Name"), blank=True, null=True, + max_length=100) + slug = models.SlugField(_(u"Slug"), unique=True, max_length=100, + blank=True, null=True) + description = models.CharField(_(u"Description"), blank=True, null=True, + max_length=500) + users = models.ManyToManyField('IshtarUser', verbose_name=_(u"Users"), + blank=True) + associated_models = models.ForeignKey( + ImporterModel, verbose_name=_(u"Associated model"), + related_name='+', blank=True, null=True) + created_models = models.ManyToManyField( + ImporterModel, verbose_name=_(u"Models that can accept new items"), + blank=True, help_text=_(u"Leave blank for no restrictions"), + related_name='+') + is_template = models.BooleanField(_(u"Is template"), default=False) + unicity_keys = models.CharField(_(u"Unicity keys (separator \";\")"), + blank=True, null=True, max_length=500) + objects = ImporterTypeManager() + + class Meta: + verbose_name = _(u"Importer - Type") + verbose_name_plural = _(u"Importer - Types") + ordering = ('name',) + + def natural_key(self): + return (self.slug, ) + + def __unicode__(self): + return self.name + + def get_importer_class(self, import_instance=None): + if self.slug and self.slug in IMPORTER_CLASSES: + cls = import_class(IMPORTER_CLASSES[self.slug]) + return cls + OBJECT_CLS = import_class(self.associated_models.klass) + DEFAULTS = dict([(default.keys, default.values) + for default in self.defaults.all()]) + LINE_FORMAT = [] + idx = 0 + for column in self.columns.order_by('col_number').all(): + idx += 1 + while column.col_number > idx: + LINE_FORMAT.append(None) + idx += 1 + targets = [] + formater_types = [] + nb = column.targets.count() + if not nb: + LINE_FORMAT.append(None) + continue + force_news = [] + concat_str = [] + for target in column.targets.all(): + ft = target.formater_type.get_formater_type( + target, import_instance=import_instance) + if not ft: + continue + formater_types.append(ft) + targets.append(target.target) + concat_str.append(target.concat_str) + force_news.append(target.force_new) + formater_kwargs = {} + if column.regexp_pre_filter: + formater_kwargs['regexp'] = re.compile( + column.regexp_pre_filter.regexp) + formater_kwargs['concat_str'] = concat_str + formater_kwargs['duplicate_fields'] = [ + (field.field_name, field.force_new, field.concat, + field.concat_str) + for field in column.duplicate_fields.all()] + formater_kwargs['label'] = column.label + formater_kwargs['required'] = column.required + formater_kwargs['force_new'] = force_news + if column.export_field_name: + formater_kwargs['export_field_name'] = [ + column.export_field_name] + formater = ImportFormater(targets, formater_types, + **formater_kwargs) + LINE_FORMAT.append(formater) + UNICITY_KEYS = [] + if self.unicity_keys: + UNICITY_KEYS = [un.strip() for un in self.unicity_keys.split(';')] + MODEL_CREATION_LIMIT = [] + for modls in self.created_models.all(): + MODEL_CREATION_LIMIT.append(import_class(modls.klass)) + args = {'OBJECT_CLS': OBJECT_CLS, 'DESC': self.description, + 'DEFAULTS': DEFAULTS, 'LINE_FORMAT': LINE_FORMAT, + 'UNICITY_KEYS': UNICITY_KEYS, + 'MODEL_CREATION_LIMIT': MODEL_CREATION_LIMIT} + name = str(''.join( + x for x in slugify(self.name).replace('-', ' ').title() + if not x.isspace())) + newclass = type(name, (Importer,), args) + return newclass + + def save(self, *args, **kwargs): + if not self.slug: + self.slug = create_slug(ImporterType, self.name) + return super(ImporterType, self).save(*args, **kwargs) + + +def get_associated_model(parent_model, keys): + model = None + if isinstance(parent_model, unicode) or \ + isinstance(parent_model, str): + OBJECT_CLS = import_class(parent_model) + else: + OBJECT_CLS = parent_model + for idx, item in enumerate(keys): + if not idx: + field = get_model_fields(OBJECT_CLS)[item] + if hasattr(field, 'rel') and hasattr(field.rel, 'to'): + model = field.rel.to + if type(field) == ModelBase: + model = field + else: + return get_associated_model(model, keys[1:]) + return model + + +class ImporterDefaultManager(models.Manager): + def get_by_natural_key(self, importer_type, target): + return self.get(importer_type__slug=importer_type, target=target) + + +class ImporterDefault(models.Model): + """ + Targets of default values in an import + """ + importer_type = models.ForeignKey(ImporterType, related_name='defaults') + target = models.CharField(u"Target", max_length=500) + + class Meta: + verbose_name = _(u"Importer - Default") + verbose_name_plural = _(u"Importer - Defaults") + unique_together = ('importer_type', 'target') + objects = ImporterDefaultManager() + + def __unicode__(self): + return u"{} - {}".format(self.importer_type, self.target) + + def natural_key(self): + return self.importer_type.slug, self.target + + @property + def keys(self): + return tuple(self.target.split('__')) + + @property + def associated_model(self): + return get_associated_model(self.importer_type.associated_models.klass, + self.keys) + + @property + def values(self): + values = {} + for default_value in self.default_values.all(): + values[default_value.target] = default_value.get_value() + return values + + +class ImporterDefaultValuesManager(models.Manager): + def get_by_natural_key(self, def_target_type, def_target, target): + return self.get(default_target__importer_type__slug=def_target_type, + default_target__target=def_target, + target=target) + + +class ImporterDefaultValues(models.Model): + """ + Default values in an import + """ + default_target = models.ForeignKey(ImporterDefault, + related_name='default_values') + target = models.CharField(u"Target", max_length=500) + value = models.CharField(u"Value", max_length=500) + objects = ImporterDefaultValuesManager() + + class Meta: + verbose_name = _(u"Importer - Default value") + verbose_name_plural = _(u"Importer - Default values") + unique_together = ('default_target', 'target') + + def natural_key(self): + return (self.default_target.importer_type.slug, + self.default_target.target, + self.target) + + def __unicode__(self): + return u"{} - {}".format(self.default_target, self.target, self.value) + + def get_value(self): + parent_model = self.default_target.associated_model + if not parent_model: + return self.value + fields = get_model_fields(parent_model) + target = self.target.strip() + if target not in fields: + return + field = fields[target] + if not hasattr(field, 'rel') or not hasattr(field.rel, 'to'): + return + model = field.rel.to + # if value is an id + try: + return model.objects.get(pk=int(self.value)) + except (ValueError, model.DoesNotExist): + pass + # try with txt_idx + try: + return model.objects.get(txt_idx=self.value) + except (ValueError, model.DoesNotExist): + pass + return "" + + +class ImporterColumnManager(models.Manager): + def get_by_natural_key(self, importer_type, col_number): + return self.get(importer_type__slug=importer_type, + col_number=col_number) + + +class ImporterColumn(models.Model): + """ + Import file column description + """ + label = models.CharField(_(u"Label"), blank=True, null=True, + max_length=200) + importer_type = models.ForeignKey(ImporterType, related_name='columns') + col_number = models.IntegerField(_(u"Column number"), default=1) + description = models.TextField(_("Description"), blank=True, null=True) + regexp_pre_filter = models.ForeignKey("Regexp", blank=True, null=True) + required = models.BooleanField(_(u"Required"), default=False) + export_field_name = models.CharField( + _(u"Export field name"), blank=True, null=True, max_length=200, + help_text=_(u"Fill this field if the field name is ambiguous for " + u"export. For instance: concatenated fields.") + ) + objects = ImporterColumnManager() + + class Meta: + verbose_name = _(u"Importer - Column") + verbose_name_plural = _(u"Importer - Columns") + ordering = ('importer_type', 'col_number') + unique_together = ('importer_type', 'col_number') + + def __unicode__(self): + return u"{} - {}".format(self.importer_type, self.col_number) + + def natural_key(self): + return self.importer_type.slug, self.col_number + + def targets_lbl(self): + return u', '.join([target.target for target in self.targets.all()]) + + def duplicate_fields_lbl(self): + return u', '.join([dp.field_name + for dp in self.duplicate_fields.all()]) + + +class ImporterDuplicateFieldManager(models.Manager): + def get_by_natural_key(self, importer_type, col_number, field_name): + return self.get(column__importer_type__slug=importer_type, + column__col_number=col_number, + field_name=field_name) + + +class ImporterDuplicateField(models.Model): + """ + Direct copy of result in other fields + """ + column = models.ForeignKey(ImporterColumn, related_name='duplicate_fields') + field_name = models.CharField(_(u"Field name"), blank=True, null=True, + max_length=200) + force_new = models.BooleanField(_(u"Force creation of new items"), + default=False) + concat = models.BooleanField(_(u"Concatenate with existing"), + default=False) + concat_str = models.CharField(_(u"Concatenate character"), max_length=5, + blank=True, null=True) + objects = ImporterDuplicateFieldManager() + + class Meta: + verbose_name = _(u"Importer - Duplicate field") + verbose_name_plural = _(u"Importer - Duplicate fields") + ordering = ('column', 'field_name') + unique_together = ('column', 'field_name') + + def natural_key(self): + return self.column.importer_type, self.column.col_number, \ + self.field_name + + +class NamedManager(models.Manager): + def get_by_natural_key(self, name): + return self.get(name=name) + + +class Regexp(models.Model): + name = models.CharField(_(u"Name"), max_length=100, unique=True) + description = models.CharField(_(u"Description"), blank=True, null=True, + max_length=500) + regexp = models.CharField(_(u"Regular expression"), max_length=500) + objects = NamedManager() + + class Meta: + verbose_name = _(u"Importer - Regular expression") + verbose_name_plural = _(u"Importer - Regular expressions") + + def __unicode__(self): + return self.name + + def natural_key(self): + return (self.name, ) + + +class ImportTargetManager(models.Manager): + def get_by_natural_key(self, importer_type, col_number, target): + return self.get(column__importer_type__slug=importer_type, + column__col_number=col_number, + target=target) + + +class ImportTarget(models.Model): + """ + Ishtar database target for a column + """ + column = models.ForeignKey(ImporterColumn, related_name='targets') + target = models.CharField(u"Target", max_length=500) + regexp_filter = models.ForeignKey("Regexp", blank=True, null=True) + formater_type = models.ForeignKey("FormaterType") + force_new = models.BooleanField(_(u"Force creation of new items"), + default=False) + concat = models.BooleanField(_(u"Concatenate with existing"), + default=False) + concat_str = models.CharField(_(u"Concatenate character"), max_length=5, + blank=True, null=True) + comment = models.TextField(_(u"Comment"), blank=True, null=True) + objects = ImportTargetManager() + + class Meta: + verbose_name = _(u"Importer - Target") + verbose_name_plural = _(u"Importer - Targets") + unique_together = ('column', 'target') + + def __unicode__(self): + return self.target[:50] if self.target else self.comment + + def natural_key(self): + return self.column.importer_type.slug, self.column.col_number, \ + self.target + + @property + def associated_model(self): + try: + return get_associated_model( + self.column.importer_type.associated_models.klass, + self.target.split('__')) + except KeyError: + return + + def get_choices(self): + if self.formater_type.formater_type == 'UnknowType' \ + and self.column.importer_type.slug: + cls = self.column.importer_type.get_importer_class() + formt = cls().line_format[self.column.col_number - 1] + if hasattr(formt.formater, 'choices'): + return [('', '--' * 8)] + list(formt.formater.choices) + return [('', '--' * 8)] + if self.formater_type.formater_type == 'StrToBoolean': + return [('', '--' * 8), + ('True', _(u"True")), + ('False', _(u"False"))] + if not self.associated_model or not hasattr(self.associated_model, + 'get_types'): + return [] + return self.associated_model.get_types() + + +class TargetKey(models.Model): + """ + User's link between import source and ishtar database. + Also temporary used for GeneralType to point missing link before adding + them in ItemKey table. + A targetkey connection can be create to be applied to on particular + import (associated_import), one particular user (associated_user) or to all + imports (associated_import and associated_user are empty). + """ + target = models.ForeignKey(ImportTarget, related_name='keys') + key = models.TextField(_(u"Key")) + value = models.TextField(_(u"Value"), blank=True, null=True) + is_set = models.BooleanField(_(u"Is set"), default=False) + associated_import = models.ForeignKey('Import', blank=True, null=True) + associated_user = models.ForeignKey('IshtarUser', blank=True, null=True) + + class Meta: + unique_together = ('target', 'key', 'associated_user', + 'associated_import') + verbose_name = _(u"Importer - Target key") + verbose_name_plural = _(u"Importer - Targets keys") + + def __unicode__(self): + return u" - ".join([unicode(self.target), self.key[:50]]) + + def column_nb(self): + # for the admin + return self.target.column.col_number + + def importer_type(self): + # for the admin + return self.target.column.importer_type.name + + def format(self): + if not self.is_set: + return None + if self.target.formater_type.formater_type == 'StrToBoolean': + if self.value in ('False', '0'): + return False + elif self.value: + return True + return + return self.value + + def save(self, *args, **kwargs): + obj = super(TargetKey, self).save(*args, **kwargs) + if not self.value: + return obj + associated_model = self.target.associated_model + if associated_model and hasattr(self.target.associated_model, + "add_key"): + v = None + # pk is given + try: + v = self.target.associated_model.objects.get( + pk=unicode(int(self.value))) + except (ValueError, self.target.associated_model.DoesNotExist): + # try with txt_idx + try: + v = self.target.associated_model.objects.get( + txt_idx=unicode(self.value)) + except self.target.associated_model.DoesNotExist: + pass + if v: + v.add_key(self.key, importer=self.associated_import) + return obj + +TARGET_MODELS = [ + ('OrganizationType', _(u"Organization type")), + ('TitleType', _(u"Title")), + ('SourceType', _(u"Source type")), + ('AuthorType', _(u"Author type")), + ('Format', _(u"Format")), + ('archaeological_operations.models.OperationType', _(u"Operation type")), + ('archaeological_operations.models.Period', _(u"Period")), + ('archaeological_operations.models.ReportState', _(u"Report state")), + ('archaeological_operations.models.RemainType', _(u"Remain type")), + ('archaeological_context_records.models.Unit', _(u"Unit")), + ('archaeological_context_records.models.ActivityType', + _(u"Activity type")), + ('archaeological_context_records.models.DocumentationType', + _(u"Documentation type")), + ('archaeological_finds.models.MaterialType', _(u"Material")), + ('archaeological_finds.models.ConservatoryState', + _(u"Conservatory state")), + ('archaeological_warehouse.models.ContainerType', _(u"Container type")), + ('archaeological_finds.models.PreservationType', _(u"Preservation type")), + ('archaeological_finds.models.ObjectType', _(u"Object type")), + ('archaeological_finds.models.IntegrityType', _(u"Integrity type")), + ('archaeological_finds.models.RemarkabilityType', + _(u"Remarkability type")), + ('archaeological_finds.models.BatchType', _(u"Batch type")), + ('archaeological_context_records.models.IdentificationType', + _("Identification type")), + ('archaeological_context_records.models.RelationType', + _(u"Context record relation type")), + ('SpatialReferenceSystem', _(u"Spatial reference system")), + ('SupportType', _(u"Support type")), + ('TitleType', _(u"Title type")), +] + +TARGET_MODELS_KEYS = [tm[0] for tm in TARGET_MODELS] + +IMPORTER_TYPES = ( + ('IntegerFormater', _(u"Integer")), + ('FloatFormater', _(u"Float")), + ('UnicodeFormater', _(u"String")), + ('DateFormater', _(u"Date")), + ('TypeFormater', _(u"Type")), + ('YearFormater', _(u"Year")), + ('StrToBoolean', _(u"String to boolean")), + ('FileFormater', pgettext_lazy("filesystem", u"File")), + ('UnknowType', _(u"Unknow type")) +) + +IMPORTER_TYPES_DCT = { + 'IntegerFormater': IntegerFormater, + 'FloatFormater': FloatFormater, + 'UnicodeFormater': UnicodeFormater, + 'DateFormater': DateFormater, + 'TypeFormater': TypeFormater, + 'YearFormater': YearFormater, + 'StrToBoolean': StrToBoolean, + 'FileFormater': FileFormater, + 'UnknowType': None, +} + +DATE_FORMATS = ( + ('%Y', _(u"4 digit year. e.g.: \"2015\"")), + ('%Y/%m/%d', _(u"4 digit year/month/day. e.g.: \"2015/02/04\"")), + ('%d/%m/%Y', _(u"Day/month/4 digit year. e.g.: \"04/02/2015\"")), +) + +IMPORTER_TYPES_CHOICES = {'TypeFormater': TARGET_MODELS, + 'DateFormater': DATE_FORMATS} + + +class FormaterTypeManager(models.Manager): + def get_by_natural_key(self, formater_type, options, many_split): + return self.get(formater_type=formater_type, + options=options, many_split=many_split) + + +class FormaterType(models.Model): + formater_type = models.CharField(u"Formater type", max_length=20, + choices=IMPORTER_TYPES) + options = models.CharField(_(u"Options"), max_length=500, blank=True, + null=True) + many_split = models.CharField(_(u"Split character(s)"), max_length=10, + blank=True, null=True) + objects = FormaterTypeManager() + + class Meta: + verbose_name = _(u"Importer - Formater type") + verbose_name_plural = _(u"Importer - Formater types") + unique_together = ('formater_type', 'options', 'many_split') + ordering = ('formater_type', 'options') + + def natural_key(self): + return self.formater_type, self.options, self.many_split + + def __unicode__(self): + return u" - ".join( + [unicode(dict(IMPORTER_TYPES)[self.formater_type]) + if self.formater_type in IMPORTER_TYPES_DCT else ''] + + [getattr(self, k) for k in ('options', 'many_split') + if getattr(self, k)]) + + def get_choices(self): + if self.format_type in IMPORTER_TYPES_CHOICES: + return IMPORTER_TYPES_CHOICES[self.format_type] + + def get_formater_type(self, target, import_instance=None): + if self.formater_type not in IMPORTER_TYPES_DCT.keys(): + return + kwargs = {'db_target': target, 'import_instance': import_instance} + if self.many_split: + kwargs['many_split'] = self.many_split + if self.formater_type == 'TypeFormater': + if self.options not in TARGET_MODELS_KEYS: + logger.warning( + "**WARN FormaterType.get_formater_type**: {} " + "is not in TARGET_MODELS_KEYS".format(self.options)) + return + model = None + if self.options in dir(): + model = dir()[self.options] + else: + model = import_class(self.options) + return TypeFormater(model, **kwargs) + elif self.formater_type == 'UnicodeFormater': + if self.options: + try: + return UnicodeFormater(int(self.options.strip()), **kwargs) + except ValueError: + pass + return UnicodeFormater(**kwargs) + elif self.formater_type == 'DateFormater': + date_formats = self.options + if self.many_split: + date_formats = self.options.split(kwargs.pop('many_split')) + return DateFormater(date_formats, **kwargs) + elif self.formater_type == 'StrToBoolean': + return StrToBoolean(**kwargs) + elif self.formater_type == 'UnknowType': + return + else: + return IMPORTER_TYPES_DCT[self.formater_type](**kwargs) + +IMPORT_STATE = (("C", _(u"Created")), + ("AP", _(u"Analyse in progress")), + ("A", _(u"Analysed")), + ("P", _(u"Import pending")), + ("IP", _(u"Import in progress")), + ("FE", _(u"Finished with errors")), + ("F", _(u"Finished")), + ("AC", _(u"Archived")), + ) + +IMPORT_STATE_DCT = dict(IMPORT_STATE) +ENCODINGS = [(settings.ENCODING, settings.ENCODING), + (settings.ALT_ENCODING, settings.ALT_ENCODING), + ('utf-8', 'utf-8')] + + +class Import(models.Model): + user = models.ForeignKey('IshtarUser') + name = models.CharField(_(u"Name"), max_length=500, + blank=True, null=True) + importer_type = models.ForeignKey(ImporterType) + imported_file = models.FileField( + _(u"Imported file"), upload_to="upload/imports/%Y/%m/", max_length=220) + imported_images = models.FileField( + _(u"Associated images (zip file)"), upload_to="upload/imports/%Y/%m/", + blank=True, null=True, max_length=220) + encoding = models.CharField(_(u"Encoding"), choices=ENCODINGS, + default=u'utf-8', max_length=15) + skip_lines = models.IntegerField(_(u"Skip lines"), default=1) + error_file = models.FileField(_(u"Error file"), + upload_to="upload/imports/%Y/%m/", + blank=True, null=True, max_length=255) + result_file = models.FileField(_(u"Result file"), + upload_to="upload/imports/%Y/%m/", + blank=True, null=True, max_length=255) + match_file = models.FileField(_(u"Match file"), + upload_to="upload/imports/%Y/%m/", + blank=True, null=True, max_length=255) + state = models.CharField(_(u"State"), max_length=2, choices=IMPORT_STATE, + default=u'C') + conservative_import = models.BooleanField( + _(u"Conservative import"), default=False, + help_text='If set to true, do not overload existing values') + creation_date = models.DateTimeField( + _(u"Creation date"), auto_now_add=True, blank=True, null=True) + end_date = models.DateTimeField(_(u"End date"), blank=True, + null=True, editable=False) + seconds_remaining = models.IntegerField( + _(u"Remaining seconds"), blank=True, null=True, editable=False) + + class Meta: + verbose_name = _(u"Import") + verbose_name_plural = _(u"Imports") + + def __unicode__(self): + return u"{} | {}".format(self.name or u"-", self.importer_type) + + def need_matching(self): + return bool(TargetKey.objects.filter(associated_import=self, + is_set=False).count()) + + @property + def errors(self): + if not self.error_file: + return [] + errors = [] + with open(self.error_file.path, 'rb') as csvfile: + reader = csv.DictReader(csvfile, fieldnames=['line', 'column', + 'error']) + reader.next() # pass the header + for row in reader: + errors.append(row) + return errors + + def get_actions(self): + """ + Get available action relevant with the current status + """ + actions = [] + if self.state == 'C': + actions.append(('A', _(u"Analyse"))) + if self.state == 'A': + actions.append(('A', _(u"Re-analyse"))) + actions.append(('I', _(u"Launch import"))) + if self.state in ('F', 'FE'): + actions.append(('A', _(u"Re-analyse"))) + actions.append(('I', _(u"Re-import"))) + actions.append(('AC', _(u"Archive"))) + if self.state == 'AC': + actions.append(('A', _(u"Unarchive"))) + actions.append(('D', _(u"Delete"))) + return actions + + @property + def imported_filename(self): + return self.imported_file.name.split(os.sep)[-1] + + @property + def status(self): + if self.state not in IMPORT_STATE_DCT: + return "" + return IMPORT_STATE_DCT[self.state] + + def get_importer_instance(self): + return self.importer_type.get_importer_class(import_instance=self)( + skip_lines=self.skip_lines, import_instance=self, + conservative_import=self.conservative_import) + + @property + def data_table(self): + imported_file = self.imported_file.path + tmpdir = None + if zipfile.is_zipfile(imported_file): + z = zipfile.ZipFile(imported_file) + filename = None + for name in z.namelist(): + # get first CSV file found + if name.endswith('.csv'): + filename = name + break + if not filename: + return [] + tmpdir = tempfile.mkdtemp(prefix='tmp-ishtar-') + imported_file = z.extract(filename, tmpdir) + + encodings = [self.encoding] + encodings += [coding for coding, c in ENCODINGS + if coding != self.encoding] + for encoding in encodings: + try: + with open(imported_file) as csv_file: + vals = [line + for line in unicodecsv.reader(csv_file, + encoding=encoding)] + if tmpdir: + shutil.rmtree(tmpdir) + return vals + except UnicodeDecodeError: + pass # try the next encoding + if tmpdir: + shutil.rmtree(tmpdir) + return [] + + def initialize(self): + self.state = 'AP' + self.save() + self.get_importer_instance().initialize(self.data_table, output='db') + self.state = 'A' + self.save() + + def importation(self): + self.state = 'IP' + self.save() + importer = self.get_importer_instance() + importer.importation(self.data_table) + # result file + filename = slugify(self.importer_type.name) + now = datetime.datetime.now().isoformat('-').replace(':', '') + result_file = filename + "_result_%s.csv" % now + self.result_file.save( + result_file, ContentFile(importer.get_csv_result().encode('utf-8'))) + + if importer.errors: + self.state = 'FE' + error_file = filename + "_errors_%s.csv" % now + self.error_file.save( + error_file, + ContentFile(importer.get_csv_errors().encode('utf-8')) + ) + else: + self.state = 'F' + self.error_file = None + if importer.match_table: + match_file = filename + "_match_%s.csv" % now + self.match_file.save( + match_file, + ContentFile(importer.get_csv_matches().encode('utf-8')) + ) + self.save() + + def archive(self): + self.state = 'AC' + self.save() + + def get_all_imported(self): + imported = [] + for related, zorg in get_all_related_m2m_objects_with_model(self): + accessor = related.get_accessor_name() + imported += [(accessor, obj) + for obj in getattr(self, accessor).all()] + return imported + + +def pre_delete_import(sender, **kwargs): + # deleted imported items when an import is delete + instance = kwargs.get('instance') + if not instance: + return + to_delete = [] + for accessor, imported in instance.get_all_imported(): + to_delete.append(imported) + for item in to_delete: + item.delete() + + +pre_delete.connect(pre_delete_import, sender=Import) |