Imports: manage control file

3 files changed, 52 insertions, 55 deletions

diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index 24c8e166d..8d53d78f1 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -35,8 +35,7 @@ class ImportFormater(object):
     def __init__(self, field_name, formater=None, required=True, through=None,
                 through_key=None, through_dict=None, through_unicity_keys=None,
                 duplicate_fields=[], regexp=None, regexp_formater_args=[],
-                reverse_for_test=None, force_value=None, post_processing=False,
-                concat=False, comment=""):
+                force_value=None, post_processing=False, concat=False, comment=""):
         self.field_name = field_name
         self.formater = formater
         self.required = required
@@ -47,7 +46,6 @@ class ImportFormater(object):
         self.duplicate_fields = duplicate_fields
         self.regexp = regexp
         self.regexp_formater_args = regexp_formater_args
-        self.reverse_for_test = reverse_for_test
         # write this value even if a value exists
         self.force_value = force_value
         # post process after import
@@ -442,26 +440,22 @@ class Importer(object):
         }
 
     def __init__(self, skip_lines=0, reference_header=None,
-             check_col_num=False, test=False, check_validity=True,
-             history_modifier=None, output='silent',
-             import_instance=None):
+             check_col_num=False, test=False, history_modifier=None,
+             output='silent', import_instance=None):
         """
          * skip_line must be set if the data provided has got headers lines.
          * a reference_header can be provided to perform a data compliance
            check. It can be useful to warn about bad parsing.
          * test doesn't write in the database
-         * check_validity rewrite a CSV file to be compared
         """
-        self.message = ''
         self.skip_lines = skip_lines
         self.reference_header = reference_header
         self.test = test
         self.errors = [] # list of (line, col, message)
-        self.messages = [] # list of (line, col, message)
+        self.validity = [] # list of (line, col, message)
         self.number_updated = 0
         self.number_created = 0
         self.check_col_num = check_col_num
-        self.check_validity = check_validity
         self.line_format = copy.copy(self.LINE_FORMAT)
         self.import_instance = import_instance
         self._defaults = self.DEFAULTS.copy()
@@ -500,17 +494,9 @@ class Importer(object):
                 formater.init(vals[idx], output)
 
     def importation(self, table, initialize=True):
-        self.validity_file = None
         if initialize:
             self.initialize(table, self.output)
-        if self.check_validity:
-            with NamedTemporaryFile(delete=False) as validity_file:
-                self.validity_file = UnicodeWriter(validity_file,
-                                delimiter=',', quotechar='"',
-                                quoting=csv.QUOTE_MINIMAL)
-                self._importation(table)
-        else:
-            self._importation(table)
+        self._importation(table)
 
     @classmethod
     def _field_name_to_data_dict(cls, field_name, value, data,
@@ -545,7 +531,7 @@ class Importer(object):
             raise ImporterError(self.ERRORS['too_many_cols'] % {
                      'user_col':len(table[0]), 'ref_col':len(self.line_format)})
         self.errors = []
-        self.messages = []
+        self.validity = []
         self.number_imported = 0
         # index of the last required column
         for idx_last_col, formater in enumerate(reversed(self.line_format)):
@@ -586,12 +572,10 @@ class Importer(object):
 
     def _line_processing(self, idx_line, line):
         if self.skip_lines > idx_line:
-            if self.validity_file:
-                self.validity_file.writerow(line)
+            self.validity.append(line)
             return
         if not line:
-            if self.validity_file:
-                self.validity_file.writerow([])
+            self.validity.append([])
             return
         self._throughs = [] # list of (formater, value)
         self._post_processing = [] # list of (formater, value)
@@ -616,8 +600,7 @@ class Importer(object):
             except:
                 pass
 
-        if self.validity_file:
-            self.validity_file.writerow(c_row)
+        self.validity.append(c_row)
         if not self.c_errors and (idx_col + 1) < self.min_col_number:
             self.c_errors = True
             self.errors.append((idx_line+1, idx_col+1,
@@ -691,8 +674,7 @@ class Importer(object):
             self._post_processing.append((formater, val))
 
         if not formater or not formater.field_name:
-            if self.validity_file:
-                c_row.append(val)
+            c_row.append('-')
             return
 
         # regex management
@@ -720,7 +702,6 @@ class Importer(object):
 
         c_values = []
         for idx_v, v in enumerate(val_group):
-            self.message = ''
             func = formater.formater
             if type(func) in (list, tuple):
                 func = func[idx_v]
@@ -746,19 +727,22 @@ class Importer(object):
                     if formater.required:
                         self.c_errors = True
                     self.errors.append((idx_line+1, idx_col+1, e.message))
-                    c_values.append(None)
+                    c_values.append('')
                     return
                 formated_values.append(value)
 
-            if self.message:
-                self.messages.append(self.message)
-
             value = formated_values
             if not many_values:
                 value = formated_values[0]
 
-            c_values.append(value)
-
+            printed_values = value
+            if type(value) not in (list, tuple):
+                printed_values = [value]
+            try:
+                # don't reunicode - unicoded values
+                c_values.append(u" ; ".join([v for v in printed_values]))
+            except TypeError:
+                c_values.append(u" ; ".join([unicode(v) for v in printed_values]))
             if value == None and formater.required:
                 self.c_errors = True
                 self.errors.append((idx_line+1, idx_col+1,
@@ -780,10 +764,7 @@ class Importer(object):
                 for field_name in field_names:
                     self._field_name_to_data_dict(field_name,
                               value, data, formater.force_value)
-        if formater.reverse_for_test:
-            c_row.append(formater.reverse_for_test(**c_values))
-        else:
-            c_row.append(unicode(c_values))
+        c_row.append(u" ; ".join([v for v in c_values]))
 
     def get_object(self, cls, data, path=[]):
         m2ms = []
@@ -850,16 +831,26 @@ class Importer(object):
             return obj, created
         return data
 
-    def get_csv_errors(self):
-        if not self.errors:
+    def _format_csv_line(self, values):
+        return u",".join([v and unicode(v).replace('"', '""') or u'-'
+                                    for v in values])
+
+    def _get_csv(self, rows, header=[]):
+        if not rows:
             return ""
-        csv_errors = ['"%s","%s","%s"' % (unicode(_("line")),
-                                unicode(_("col")), unicode(_("error")))]
-        for line, col, error in self.errors:
-            csv_errors.append(u'"%s","%s","%s"' % (line and unicode(line) or '-',
-                                                   col and unicode(col) or '-',
-                                                        unicode(error)))
-        return u"\n".join(csv_errors)
+        csv_v = []
+        if header:
+            csv_v.append(self._format_csv_line(header))
+        for values in rows:
+            csv_v.append(self._format_csv_line(values))
+        return u"\n".join(csv_v)
+
+    def get_csv_errors(self):
+        return self._get_csv(self.errors,
+                            header=[_("line"), _("col"), _("error")])
+
+    def get_csv_result(self):
+        return self._get_csv(self.validity)
 
     @classmethod
     def choices_check(cls, choices):
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index 20b85e614..c7f89299e 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -1400,15 +1400,18 @@ class Import(models.Model):
         self.save()
         importer = self.get_importer_instance()
         importer.importation(self.data_table)
+        # result file
+        filename = slugify(self.importer_type.name)
+        now = datetime.datetime.now().isoformat('-').replace(':','')
+        result_file = filename + "_result_%s.csv" % now
+        result_file = os.sep.join([self.result_file.storage.location,
+                                   result_file])
+        with open(result_file, 'w') as fle:
+            fle.write(importer.get_csv_result().encode('utf-8'))
+        self.result_file = File(open(fle.name))
         if importer.errors:
             self.state = 'FE'
-            error_file = None
-
-            filename = slugify(self.importer_type.name) + ".csv"
-            now = datetime.datetime.now().isoformat('-'
-                                        ).replace(':','')
-            error_file = '.'.join(filename.split('.')[:-1]) \
-                                     + "_errors_%s.csv" % now
+            error_file = filename + "_errors_%s.csv" % now
             error_file = os.sep.join([self.error_file.storage.location,
                                       error_file])
             with open(error_file, 'w') as fle:
diff --git a/ishtar_common/templates/ishtar/import_list.html b/ishtar_common/templates/ishtar/import_list.html
index 40bc019ad..9e9b4cb61 100644
--- a/ishtar_common/templates/ishtar/import_list.html
+++ b/ishtar_common/templates/ishtar/import_list.html
@@ -46,6 +46,9 @@
   <td>{% if import.error_file %}
     <a href='{{MEDIA_URL}}{{import.error_file}}'>{% trans "Error file" %}</a>
   {% endif %}</td>
+  <td>{% if import.result_file %}
+    <a href='{{MEDIA_URL}}{{import.result_file}}'>{% trans "Control file" %}</a>
+  {% endif %}</td>
 </tr>
 {% endfor %}
 </table>