Import: add management of link between user data and db data

1 files changed, 85 insertions, 21 deletions

diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index f2193eae2..1f059f2f5 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -28,6 +28,7 @@ from importlib import import_module
 import os
 import re
 import tempfile
+import unicodecsv
 
 from django.conf import settings
 from django.core.cache import cache
@@ -955,13 +956,6 @@ class OrganizationType(GeneralType):
         verbose_name_plural = _(u"Organization types")
         ordering = ('label',)
 
-IMPORT_STATE = (("C", _(u"Created")),
-                ("AP", _(u"Analyse in progress")),
-                ("A", _(u"Analysed")),
-                ("P", _(u"Import pending")),
-                ("IP", _(u"Import in progress")),
-                ("F", _(u"Finished")))
-
 MODELS = [
           ('archaeological_operations.models.Operation', _(u"Operation")),
           ('archaeological_operations.models.Parcel', _(u"Parcels")),
@@ -985,6 +979,9 @@ def get_model_fields(model):
     return fields
 
 class ImporterType(models.Model):
+    """
+    Description of a table to be mapped with ishtar database
+    """
     name = models.CharField(_(u"Name"), blank=True, null=True,
                             max_length=100)
     description = models.CharField(_(u"Description"), blank=True, null=True,
@@ -1022,7 +1019,7 @@ class ImporterType(models.Model):
                 LINE_FORMAT.append(None)
                 continue
             for target in column.targets.all():
-                ft = target.formater_type.get_formater_type()
+                ft = target.formater_type.get_formater_type(target)
                 if not ft:
                     continue
                 formater_types.append(ft)
@@ -1032,8 +1029,9 @@ class ImporterType(models.Model):
                 formater_kwargs['regexp'] = re.compile(
                                                 column.regexp_pre_filter.regexp)
             formater_kwargs['duplicate_fields'] = [field.field_name
-                                        for field in column.duplicate_fields.all()]
-            formater = ImportFormater(targets, formater_types, **formater_kwargs)
+                                     for field in column.duplicate_fields.all()]
+            formater = ImportFormater(targets, formater_types,
+                                      **formater_kwargs)
             LINE_FORMAT.append(formater)
         args = {'OBJECT_CLS':OBJECT_CLS, 'DESC':self.description,
                 'DEFAULTS':DEFAULTS, 'LINE_FORMAT':LINE_FORMAT}
@@ -1041,6 +1039,9 @@ class ImporterType(models.Model):
         return newclass
 
 class ImporterDefault(models.Model):
+    """
+    Targets of default values in an import
+    """
     importer_type = models.ForeignKey(ImporterType, related_name='defaults')
     target = models.CharField(u"Target", max_length=500)
     class Meta:
@@ -1071,6 +1072,9 @@ class ImporterDefault(models.Model):
         return values
 
 class ImporterDefaultValues(models.Model):
+    """
+    Default values in an import
+    """
     default_target = models.ForeignKey(ImporterDefault,
                                        related_name='default_values')
     target = models.CharField(u"Target", max_length=500)
@@ -1096,6 +1100,9 @@ class ImporterDefaultValues(models.Model):
         return ""
 
 class ImporterColumn(models.Model):
+    """
+    Import file column description
+    """
     importer_type = models.ForeignKey(ImporterType, related_name='columns')
     col_number = models.IntegerField(_(u"Column number"), default=1)
     regexp_pre_filter = models.ForeignKey("Regexp", blank=True, null=True)
@@ -1105,6 +1112,9 @@ class ImporterColumn(models.Model):
         verbose_name_plural = _(u"Importer - Columns")
 
 class ImporterDuplicateField(models.Model):
+    """
+    Direct copy of result in other fields
+    """
     column = models.ForeignKey(ImporterColumn, related_name='duplicate_fields')
     field_name = models.CharField(_(u"Field name"), blank=True, null=True,
                                   max_length=200)
@@ -1124,6 +1134,9 @@ class Regexp(models.Model):
 IMPORTER_TYPES = []
 
 class ImportTarget(models.Model):
+    """
+    Ishtar database target for a column
+    """
     column = models.ForeignKey(ImporterColumn, related_name='targets')
     target = models.CharField(u"Target", max_length=500)
     regexp_filter = models.ForeignKey("Regexp", blank=True, null=True)
@@ -1132,6 +1145,26 @@ class ImportTarget(models.Model):
         verbose_name = _(u"Importer - Target")
         verbose_name_plural = _(u"Importer - Targets")
 
+    def __unicode__(self):
+        return u" - ".join([unicode(self.column), self.target[:50]])
+
+class TargetKey(models.Model):
+    """
+    User's link between import source and ishtar database.
+    Also temporary used for GeneralType to point missing link before adding
+    them in ItemKey table
+    """
+    target = models.ForeignKey(ImporterColumn, related_name='keys')
+    key = models.TextField(_(u"Key"), blank=True, null=True)
+    value = models.TextField(_(u"Value"))
+    is_set = models.BooleanField(_(u"Is set"), default=False)
+
+    class Meta:
+        unique_together = ('target', 'value')
+
+    def __unicode__(self):
+        return u" - ".join([unicode(self.target), self.key[:50]])
+
 TARGET_MODELS = [
     ('OrganizationType', _(u"Organization type")),
     ('SourceType', _(u"Source type")),
@@ -1190,10 +1223,10 @@ class FormaterType(models.Model):
         if self.format_type in IMPORTER_TYPES_CHOICES:
             return IMPORTER_TYPES_CHOICES[self.format_type]
 
-    def get_formater_type(self):
+    def get_formater_type(self, target):
         if self.formater_type not in IMPORTER_TYPES_DCT.keys():
             return
-        kwargs = {}
+        kwargs = {'target':target}
         if self.many_split:
             kwargs['many_split'] = self.many_split
         if self.formater_type == 'TypeFormater':
@@ -1206,35 +1239,44 @@ class FormaterType(models.Model):
                 model = import_module(self.options)
             return TypeFormater(model, **kwargs)
         elif self.formater_type == 'IntegerFormater':
-            return IntegerFormater()
+            return IntegerFormater(**kwargs)
         elif self.formater_type == 'FloatFormater':
-            return FloatFormater()
+            return FloatFormater(**kwargs)
         elif self.format_type == 'UnicodeFormater':
             try:
-                return UnicodeFormater(int(self.options.strip()))
+                return UnicodeFormater(int(self.options.strip()), **kwargs)
             except ValueError:
                 return
         elif self.format_type == 'DateFormater':
-            return DateFormater(self.options)
+            return DateFormater(self.options, **kwargs)
+
+IMPORT_STATE = (("C", _(u"Created")),
+                ("AP", _(u"Analyse in progress")),
+                ("A", _(u"Analysed")),
+                ("P", _(u"Import pending")),
+                ("IP", _(u"Import in progress")),
+                ("F", _(u"Finished")))
 
 class Import(models.Model):
     user = models.ForeignKey('IshtarUser')
     importer_type = models.ForeignKey(ImporterType)
     imported_file = models.FileField(_(u"Imported file"),
                                      upload_to="upload/imports/")
+    skip_lines = models.IntegerField(default=1)
     error_file = models.FileField(_(u"Error file"),
                                      upload_to="upload/imports/",
                                      blank=True, null=True)
     result_file = models.FileField(_(u"Result file"),
                                      upload_to="upload/imports/",
                                      blank=True, null=True)
-    state = models.CharField(_(u"State"), max_length=2, choices=IMPORT_STATE)
-    creation_date = models.DateTimeField(_(u"Creation date"), blank=True,
-                                         null=True)
+    state = models.CharField(_(u"State"), max_length=2, choices=IMPORT_STATE,
+                             default='C')
+    creation_date = models.DateTimeField(_(u"Creation date"), auto_now_add=True,
+                                         blank=True, null=True)
     end_date = models.DateTimeField(_(u"End date"), blank=True,
-                                    null=True)
+                                    null=True, editable=False)
     seconds_remaining = models.IntegerField(_(u"Seconds remaining"), blank=True,
-                                            null=True)
+                                            null=True, editable=False)
     class Meta:
         verbose_name = _(u"Import")
         verbose_name_plural = _(u"Imports")
@@ -1243,6 +1285,28 @@ class Import(models.Model):
         return u"%s - %s" % (unicode(self.importer_type),
                              unicode(self.user))
 
+    @property
+    def importer_instance(self):
+        return self.importer_type.importer_class(skip_lines=self.skip_lines,
+                                                 import_instance=self)
+
+    @property
+    def data_table(self):
+        encodings = [settings.ENCODING, settings.ALT_ENCODING, 'utf-8']
+        with open(self.imported_file.filename) as csv_file:
+            for encoding in encodings:
+                try:
+                    return [line for line in unicodecsv.reader(csv_file,
+                                                       encoding=encoding)]
+                except UnicodeDecodeError:
+                    if encoding != encodings[-1]:
+                        csv_file.seek(0)
+                        continue
+        return []
+
+    def initialize(self):
+        self.importer_instance.initialize(self.data_table, output='db')
+
 class Organization(Address, Merge, OwnPerms, ValueGetter):
     TABLE_COLS = ('name', 'organization_type',)
     name = models.CharField(_(u"Name"), max_length=300)