diff options
Diffstat (limited to 'ishtar_common/models.py')
| -rw-r--r-- | ishtar_common/models.py | 273 |
1 files changed, 262 insertions, 11 deletions
diff --git a/ishtar_common/models.py b/ishtar_common/models.py index 6c153fb9c..f2193eae2 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -21,11 +21,13 @@ Models description """ from cStringIO import StringIO +import copy import datetime from PIL import Image +from importlib import import_module import os +import re import tempfile -import copy from django.conf import settings from django.core.cache import cache @@ -51,6 +53,8 @@ from simple_history.models import HistoricalRecords as BaseHistoricalRecords from ishtar_common.ooo_replace import ooo_replace from ishtar_common.model_merging import merge_model_objects from ishtar_common.utils import get_cache +from ishtar_common.data_importer import Importer, ImportFormater, IntegerFormater, \ + FloatFormater, UnicodeFormater, DateFormater, TypeFormater def post_save_user(sender, **kwargs): user = kwargs['instance'] @@ -958,16 +962,264 @@ IMPORT_STATE = (("C", _(u"Created")), ("IP", _(u"Import in progress")), ("F", _(u"Finished"))) -IMPORT_TYPE = ( - ('archaeological_files.data_importer.FileImporterSraPdL', - _("Archaeological files - SRA Pays de la Loire")), - ('custom', _(u"Custom")) - ) - -class Import(models.Model): +MODELS = [ + ('archaeological_operations.models.Operation', _(u"Operation")), + ('archaeological_operations.models.Parcel', _(u"Parcels")), + ('archaeological_operations.models.OperationSource', + _(u"Operation source")), + ] +if 'archaeological_files' in settings.INSTALLED_APPS: + MODELS = [('archaeological_files.models.File', _(u"Archaeological files")), + ] + MODELS + +def get_model_fields(model): + """ + Return a dict of fields from model + To be replace in Django 1.8 with get_fields, get_field + """ + fields = {} + options = model._meta + for field in sorted(options.concrete_fields + options.many_to_many + + options.virtual_fields): + fields[field.name] = field + return fields + +class ImporterType(models.Model): name = models.CharField(_(u"Name"), blank=True, null=True, max_length=100) + description = models.CharField(_(u"Description"), blank=True, null=True, + max_length=500) + users = models.ManyToManyField('IshtarUser', verbose_name=_(u"Users"), + blank=True, null=True) + associated_models = models.CharField(_(u"Associated model"), max_length=200, + choices=MODELS) + is_template = models.BooleanField(_(u"Is template"), default=False) + class Meta: + verbose_name = _(u"Importer - Type") + verbose_name_plural = _(u"Importer - Types") + + def __unicode__(self): + return self.name + + @property + def importer_class(self): + name = ''.join(x for x in slugify(self.name).replace('-', ' ').title() + if not x.isspace()) + OBJECT_CLS = import_module(self.associated_models) + DEFAULTS = dict((default.keys, default.values) + for default in self.defaults.all()) + LINE_FORMAT = [] + idx = 0 + for column in self.columns.order_by('col_number').all(): + idx += 1 + while column.order > idx: + LINE_FORMAT.append(None) + idx += 1 + targets = None + formater_types = None + nb = column.targets.count() + if not nb: + LINE_FORMAT.append(None) + continue + for target in column.targets.all(): + ft = target.formater_type.get_formater_type() + if not ft: + continue + formater_types.append(ft) + targets.append(target.target) + formater_kwargs = {} + if column.regexp_pre_filter: + formater_kwargs['regexp'] = re.compile( + column.regexp_pre_filter.regexp) + formater_kwargs['duplicate_fields'] = [field.field_name + for field in column.duplicate_fields.all()] + formater = ImportFormater(targets, formater_types, **formater_kwargs) + LINE_FORMAT.append(formater) + args = {'OBJECT_CLS':OBJECT_CLS, 'DESC':self.description, + 'DEFAULTS':DEFAULTS, 'LINE_FORMAT':LINE_FORMAT} + newclass = type(name, (Importer,), args) + return newclass + +class ImporterDefault(models.Model): + importer_type = models.ForeignKey(ImporterType, related_name='defaults') + target = models.CharField(u"Target", max_length=500) + class Meta: + verbose_name = _(u"Importer - Default") + verbose_name_plural = _(u"Importer - Defaults") + + @property + def keys(self): + return default.target.split('__') + + @property + def associated_model(self): + field = None + OBJECT_CLS = import_module(self.importer_type.associated_models) + for idx, item in enumerate(self.keys): + if not idx: + field = get_model_fields(OBJECT_CLS)[item] + else: + raise NotImplemented() + if hasattr(field, 'rel') and hasattr(field.rel, 'to'): + return field.rel.to + + @property + def values(self): + values = {} + for default_value in self.default_values.all(): + values[default_value.target] = default_value.get_value() + return values + +class ImporterDefaultValues(models.Model): + default_target = models.ForeignKey(ImporterDefault, + related_name='default_values') + target = models.CharField(u"Target", max_length=500) + value = models.CharField(u"Value", max_length=500) + class Meta: + verbose_name = _(u"Importer - Default value") + verbose_name_plural = _(u"Importer - Default values") + + def get_value(self): + model = self.default_target.associated_model + if not model: + return self.value + # if value is an id + try: + return model.objects.get(pk=int(self.value)) + except (ValueError, model.DoesNotExist): + pass + # try with txt_idx + try: + return model.objects.get(txt_idx=self.value) + except (ValueError, model.DoesNotExist): + pass + return "" + +class ImporterColumn(models.Model): + importer_type = models.ForeignKey(ImporterType, related_name='columns') + col_number = models.IntegerField(_(u"Column number"), default=1) + regexp_pre_filter = models.ForeignKey("Regexp", blank=True, null=True) + required = models.BooleanField(_(u"Required"), default=False) + class Meta: + verbose_name = _(u"Importer - Column") + verbose_name_plural = _(u"Importer - Columns") + +class ImporterDuplicateField(models.Model): + column = models.ForeignKey(ImporterColumn, related_name='duplicate_fields') + field_name = models.CharField(_(u"Field name"), blank=True, null=True, + max_length=200) + class Meta: + verbose_name = _(u"Importer - Duplicate field") + verbose_name_plural = _(u"Importer - Duplicate fields") + +class Regexp(models.Model): + name = models.CharField(_(u"Name"), max_length=100) + description = models.CharField(_(u"Description"), blank=True, null=True, + max_length=500) + regexp = models.CharField(_(u"Regular expression"), max_length=500) + class Meta: + verbose_name = _(u"Importer - Regular expression") + verbose_name_plural = _(u"Importer - Regular expressions") + +IMPORTER_TYPES = [] + +class ImportTarget(models.Model): + column = models.ForeignKey(ImporterColumn, related_name='targets') + target = models.CharField(u"Target", max_length=500) + regexp_filter = models.ForeignKey("Regexp", blank=True, null=True) + formater_type = models.ForeignKey("FormaterType") + class Meta: + verbose_name = _(u"Importer - Target") + verbose_name_plural = _(u"Importer - Targets") + +TARGET_MODELS = [ + ('OrganizationType', _(u"Organization type")), + ('SourceType', _(u"Source type")), + ('AuthorType', _(u"Author type")), + ('Format', _(u"Format")), + ('archaeological_operations.models.OperationType', _(u"OperationType")), + ('archaeological_operations.models.Period', _(u"Period")), + ] + +TARGET_MODELS_KEYS = (tm[0] for tm in TARGET_MODELS) + +IMPORTER_TYPES = ( + ('IntegerFormater', _(u"Integer")), + ('FloatFormater', _(u"Float")), + ('UnicodeFormater', _(u"String")), + ('DateFormater', _(u"Date")), + ('TypeFormater', _(u"Type")), +) + +IMPORTER_TYPES_DCT = { + 'IntegerFormater':IntegerFormater, + 'FloatFormater':FloatFormater, + 'UnicodeFormater':UnicodeFormater, + 'DateFormater':DateFormater, + 'TypeFormater':TypeFormater, +} + +DATE_FORMATS = ( + ('%Y', _(u"4 digit year. e.g.: \"2015\"")), + ('%Y/%m/%d', _(u"4 digit year/month/day. e.g.: \"2015/02/04\"")), + ('%d/%m/%Y', _(u"Day/month/4 digit year. e.g.: \"04/02/2015\"")), +) + +IMPORTER_TYPES_CHOICES = {'TypeFormater':TARGET_MODELS, + 'DateFormater':DATE_FORMATS} + +class FormaterType(models.Model): + formater_type = models.CharField(u"Formater type", max_length=20, + choices=IMPORTER_TYPES) + options = models.CharField(_(u"Options"), max_length=500, blank=True, + null=True) + many_split = models.CharField(_(u"Split character(s)"), max_length=10, + blank=True, null=True) + class Meta: + verbose_name = _(u"Importer - Formater type") + verbose_name_plural = _(u"Importer - Formater types") + unique_together = ('formater_type', 'options', 'many_split') + + def __unicode__(self): + return u" - ".join([unicode(dict(IMPORTER_TYPES)[self.formater_type]) + if self.formater_type in IMPORTER_TYPES_DCT else ''] + + [getattr(self, k) for k in ('options', 'many_split') + if getattr(self, k)]) + + def get_choices(self): + if self.format_type in IMPORTER_TYPES_CHOICES: + return IMPORTER_TYPES_CHOICES[self.format_type] + + def get_formater_type(self): + if self.formater_type not in IMPORTER_TYPES_DCT.keys(): + return + kwargs = {} + if self.many_split: + kwargs['many_split'] = self.many_split + if self.formater_type == 'TypeFormater': + if self.options not in TARGET_MODELS_KEYS: + return + model = None + if self.options in dir(): + model = dir()[self.options] + else: + model = import_module(self.options) + return TypeFormater(model, **kwargs) + elif self.formater_type == 'IntegerFormater': + return IntegerFormater() + elif self.formater_type == 'FloatFormater': + return FloatFormater() + elif self.format_type == 'UnicodeFormater': + try: + return UnicodeFormater(int(self.options.strip())) + except ValueError: + return + elif self.format_type == 'DateFormater': + return DateFormater(self.options) + +class Import(models.Model): user = models.ForeignKey('IshtarUser') + importer_type = models.ForeignKey(ImporterType) imported_file = models.FileField(_(u"Imported file"), upload_to="upload/imports/") error_file = models.FileField(_(u"Error file"), @@ -976,8 +1228,6 @@ class Import(models.Model): result_file = models.FileField(_(u"Result file"), upload_to="upload/imports/", blank=True, null=True) - importer_type = models.CharField(_(u"Importer type"), max_length=200, - choices=IMPORT_TYPE) state = models.CharField(_(u"State"), max_length=2, choices=IMPORT_STATE) creation_date = models.DateTimeField(_(u"Creation date"), blank=True, null=True) @@ -990,7 +1240,8 @@ class Import(models.Model): verbose_name_plural = _(u"Imports") def __unicode__(self): - return self.name + return u"%s - %s" % (unicode(self.importer_type), + unicode(self.user)) class Organization(Address, Merge, OwnPerms, ValueGetter): TABLE_COLS = ('name', 'organization_type',) |