diff options
Diffstat (limited to 'ishtar_common/models.py')
| -rw-r--r-- | ishtar_common/models.py | 106 |
1 files changed, 85 insertions, 21 deletions
diff --git a/ishtar_common/models.py b/ishtar_common/models.py index f2193eae2..1f059f2f5 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -28,6 +28,7 @@ from importlib import import_module import os import re import tempfile +import unicodecsv from django.conf import settings from django.core.cache import cache @@ -955,13 +956,6 @@ class OrganizationType(GeneralType): verbose_name_plural = _(u"Organization types") ordering = ('label',) -IMPORT_STATE = (("C", _(u"Created")), - ("AP", _(u"Analyse in progress")), - ("A", _(u"Analysed")), - ("P", _(u"Import pending")), - ("IP", _(u"Import in progress")), - ("F", _(u"Finished"))) - MODELS = [ ('archaeological_operations.models.Operation', _(u"Operation")), ('archaeological_operations.models.Parcel', _(u"Parcels")), @@ -985,6 +979,9 @@ def get_model_fields(model): return fields class ImporterType(models.Model): + """ + Description of a table to be mapped with ishtar database + """ name = models.CharField(_(u"Name"), blank=True, null=True, max_length=100) description = models.CharField(_(u"Description"), blank=True, null=True, @@ -1022,7 +1019,7 @@ class ImporterType(models.Model): LINE_FORMAT.append(None) continue for target in column.targets.all(): - ft = target.formater_type.get_formater_type() + ft = target.formater_type.get_formater_type(target) if not ft: continue formater_types.append(ft) @@ -1032,8 +1029,9 @@ class ImporterType(models.Model): formater_kwargs['regexp'] = re.compile( column.regexp_pre_filter.regexp) formater_kwargs['duplicate_fields'] = [field.field_name - for field in column.duplicate_fields.all()] - formater = ImportFormater(targets, formater_types, **formater_kwargs) + for field in column.duplicate_fields.all()] + formater = ImportFormater(targets, formater_types, + **formater_kwargs) LINE_FORMAT.append(formater) args = {'OBJECT_CLS':OBJECT_CLS, 'DESC':self.description, 'DEFAULTS':DEFAULTS, 'LINE_FORMAT':LINE_FORMAT} @@ -1041,6 +1039,9 @@ class ImporterType(models.Model): return newclass class ImporterDefault(models.Model): + """ + Targets of default values in an import + """ importer_type = models.ForeignKey(ImporterType, related_name='defaults') target = models.CharField(u"Target", max_length=500) class Meta: @@ -1071,6 +1072,9 @@ class ImporterDefault(models.Model): return values class ImporterDefaultValues(models.Model): + """ + Default values in an import + """ default_target = models.ForeignKey(ImporterDefault, related_name='default_values') target = models.CharField(u"Target", max_length=500) @@ -1096,6 +1100,9 @@ class ImporterDefaultValues(models.Model): return "" class ImporterColumn(models.Model): + """ + Import file column description + """ importer_type = models.ForeignKey(ImporterType, related_name='columns') col_number = models.IntegerField(_(u"Column number"), default=1) regexp_pre_filter = models.ForeignKey("Regexp", blank=True, null=True) @@ -1105,6 +1112,9 @@ class ImporterColumn(models.Model): verbose_name_plural = _(u"Importer - Columns") class ImporterDuplicateField(models.Model): + """ + Direct copy of result in other fields + """ column = models.ForeignKey(ImporterColumn, related_name='duplicate_fields') field_name = models.CharField(_(u"Field name"), blank=True, null=True, max_length=200) @@ -1124,6 +1134,9 @@ class Regexp(models.Model): IMPORTER_TYPES = [] class ImportTarget(models.Model): + """ + Ishtar database target for a column + """ column = models.ForeignKey(ImporterColumn, related_name='targets') target = models.CharField(u"Target", max_length=500) regexp_filter = models.ForeignKey("Regexp", blank=True, null=True) @@ -1132,6 +1145,26 @@ class ImportTarget(models.Model): verbose_name = _(u"Importer - Target") verbose_name_plural = _(u"Importer - Targets") + def __unicode__(self): + return u" - ".join([unicode(self.column), self.target[:50]]) + +class TargetKey(models.Model): + """ + User's link between import source and ishtar database. + Also temporary used for GeneralType to point missing link before adding + them in ItemKey table + """ + target = models.ForeignKey(ImporterColumn, related_name='keys') + key = models.TextField(_(u"Key"), blank=True, null=True) + value = models.TextField(_(u"Value")) + is_set = models.BooleanField(_(u"Is set"), default=False) + + class Meta: + unique_together = ('target', 'value') + + def __unicode__(self): + return u" - ".join([unicode(self.target), self.key[:50]]) + TARGET_MODELS = [ ('OrganizationType', _(u"Organization type")), ('SourceType', _(u"Source type")), @@ -1190,10 +1223,10 @@ class FormaterType(models.Model): if self.format_type in IMPORTER_TYPES_CHOICES: return IMPORTER_TYPES_CHOICES[self.format_type] - def get_formater_type(self): + def get_formater_type(self, target): if self.formater_type not in IMPORTER_TYPES_DCT.keys(): return - kwargs = {} + kwargs = {'target':target} if self.many_split: kwargs['many_split'] = self.many_split if self.formater_type == 'TypeFormater': @@ -1206,35 +1239,44 @@ class FormaterType(models.Model): model = import_module(self.options) return TypeFormater(model, **kwargs) elif self.formater_type == 'IntegerFormater': - return IntegerFormater() + return IntegerFormater(**kwargs) elif self.formater_type == 'FloatFormater': - return FloatFormater() + return FloatFormater(**kwargs) elif self.format_type == 'UnicodeFormater': try: - return UnicodeFormater(int(self.options.strip())) + return UnicodeFormater(int(self.options.strip()), **kwargs) except ValueError: return elif self.format_type == 'DateFormater': - return DateFormater(self.options) + return DateFormater(self.options, **kwargs) + +IMPORT_STATE = (("C", _(u"Created")), + ("AP", _(u"Analyse in progress")), + ("A", _(u"Analysed")), + ("P", _(u"Import pending")), + ("IP", _(u"Import in progress")), + ("F", _(u"Finished"))) class Import(models.Model): user = models.ForeignKey('IshtarUser') importer_type = models.ForeignKey(ImporterType) imported_file = models.FileField(_(u"Imported file"), upload_to="upload/imports/") + skip_lines = models.IntegerField(default=1) error_file = models.FileField(_(u"Error file"), upload_to="upload/imports/", blank=True, null=True) result_file = models.FileField(_(u"Result file"), upload_to="upload/imports/", blank=True, null=True) - state = models.CharField(_(u"State"), max_length=2, choices=IMPORT_STATE) - creation_date = models.DateTimeField(_(u"Creation date"), blank=True, - null=True) + state = models.CharField(_(u"State"), max_length=2, choices=IMPORT_STATE, + default='C') + creation_date = models.DateTimeField(_(u"Creation date"), auto_now_add=True, + blank=True, null=True) end_date = models.DateTimeField(_(u"End date"), blank=True, - null=True) + null=True, editable=False) seconds_remaining = models.IntegerField(_(u"Seconds remaining"), blank=True, - null=True) + null=True, editable=False) class Meta: verbose_name = _(u"Import") verbose_name_plural = _(u"Imports") @@ -1243,6 +1285,28 @@ class Import(models.Model): return u"%s - %s" % (unicode(self.importer_type), unicode(self.user)) + @property + def importer_instance(self): + return self.importer_type.importer_class(skip_lines=self.skip_lines, + import_instance=self) + + @property + def data_table(self): + encodings = [settings.ENCODING, settings.ALT_ENCODING, 'utf-8'] + with open(self.imported_file.filename) as csv_file: + for encoding in encodings: + try: + return [line for line in unicodecsv.reader(csv_file, + encoding=encoding)] + except UnicodeDecodeError: + if encoding != encodings[-1]: + csv_file.seek(0) + continue + return [] + + def initialize(self): + self.importer_instance.initialize(self.data_table, output='db') + class Organization(Address, Merge, OwnPerms, ValueGetter): TABLE_COLS = ('name', 'organization_type',) name = models.CharField(_(u"Name"), max_length=300) |