Work on SRA importation

1 files changed, 379 insertions, 162 deletions

diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index 86285e33e..1d768c6b0 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -17,139 +17,24 @@
 
 # See the file COPYING for details.
 
-"""
-# Usage exemple (extracted from simulabio application)
-
-class HarvestPlotImporter(Importer):
-    LINE_FORMAT = [
-               ImportFormater('name', Importer.get_unicode_formater(100)),
-               ImportFormater('plot_group_number',
-                              Importer.get_unicode_formater(3), required=False),
-               ImportFormater('geographical_area', unicode, required=False),
-               ImportFormater('soil_type', Importer.choices_check(SOIL_TYPE),
-                                                            required=False),
-               ImportFormater('cow_access', Importer.boolean_formater),
-               ImportFormater('area', Importer.float_formater),
-               ImportFormater('remark', unicode, required=False),
-               ImportFormater('diagnostic', Importer.boolean_formater),
-               ImportFormater('project', Importer.boolean_formater),
-               ImportFormater('harvest_n2', 'harvest_formater', required=False),
-               ImportFormater('harvest_n1', 'harvest_formater', required=False),
-               ImportFormater('harvest', 'harvest_formater'),
-               ImportFormater('harvest_setting', 'harvest_formater',
-                              through=HarvestTransition,
-                              through_key='plot',
-                              through_dict={'year':1},
-                              through_unicity_keys=['plot', 'year'],
-                              required=False),
-               ImportFormater('harvest_setting', 'harvest_formater',
-                              through=HarvestTransition,
-                              through_key='plot',
-                              through_dict={'year':2},
-                              through_unicity_keys=['plot', 'year'],
-                              required=False),
-               ImportFormater('harvest_setting', 'harvest_formater',
-                              through=HarvestTransition,
-                              through_key='plot',
-                              through_dict={'year':3},
-                              through_unicity_keys=['plot', 'year'],
-                              required=False),
-               ImportFormater('harvest_setting', 'harvest_formater',
-                              through=HarvestTransition,
-                              through_key='plot',
-                              through_dict={'year':4},
-                              through_unicity_keys=['plot', 'year'],
-                              required=False),
-               ImportFormater('harvest_setting', 'harvest_formater',
-                              through=HarvestTransition,
-                              through_key='plot',
-                              through_dict={'year':5},
-                              through_unicity_keys=['plot', 'year'],
-                              required=False),
-               ImportFormater('harvest_setting', 'harvest_formater',
-                              through=HarvestTransition,
-                              through_key='plot',
-                              through_dict={'year':6},
-                              through_unicity_keys=['plot', 'year'],
-                              required=False),
-               ]
-    OBJECT_CLS = HarvestPlots
-    UNICITY_KEYS = []
-
-    def __init__(self, study, skip_first_line=None):
-        # get the reference header
-        dct = {'separator':settings.CSV_DELIMITER}
-        dct['data'] = Harvest.objects.filter(available=True).all()
-        reference_file = render_to_string('simulabio/files/parcelles_ref.csv',
-                                          dct)
-        reference_header = unicode_csv_reader(
-                                [reference_file.split('\n')[0]]).next()
-        super(HarvestPlotImporter, self).__init__(
-                                           skip_first_line=skip_first_line,
-                                           reference_header=reference_header)
-        self.study = study
-        self.default_vals = {'study':self.study}
-
-    def harvest_formater(self, value):
-        value = value.strip()
-        if not value:
-            return
-        try:
-            harvest = Harvest.objects.get(name__iexact=value)
-        except ObjectDoesNotExist:
-            raise ValueError(_(u"\"%(value)s\" not in %(values)s") % {
-                'value':value,
-                'values':u", ".join([val.name
-                        for val in Harvest.objects.filter(available=True)])
-                })
-        hs, created = HarvestSettings.objects.get_or_create(study=self.study,
-                                                            harvest=harvest)
-        if created:
-            self.message = _(u"\"%(harvest)s\" has been added in your settings "
-                             u"don't forget to fill yields for this harvest.") \
-                             % {'harvest':harvest.name}
-        return hs
-
-class HarvestPlotsImportForm(forms.Form):
-    csv_file = forms.FileField(label=_(u"Plot list file (CSV)"))
-
-    def save(self, study):
-        csv_file = self.cleaned_data['csv_file']
-        importer = models.HarvestPlotImporter(study, skip_first_line=True)
-        # some softwares (at least Gnumeric) convert CSV file to utf-8 no matter
-        # what the CSV source encoding is
-        encodings = [settings.ENCODING, 'utf-8']
-        for encoding in encodings:
-            try:
-                importer.importation(unicode_csv_reader(
-                                [line.decode(encoding)
-                                 for line in csv_file.readlines()]))
-            except ImporterError, e:
-                if e.type == ImporterError.HEADER and encoding != encodings[-1]:
-                    csv_file.seek(0)
-                    continue
-                return 0, [[0, 0, e.msg]], []
-            except UnicodeDecodeError, e:
-                return 0, [[0, 0, Importer.ERRORS['header_check']]], []
-            break
-        return importer.number_imported, importer.errors, importer.messages
-"""
-
 import copy, csv, datetime, logging, sys
 from tempfile import NamedTemporaryFile
 
 from django.contrib.auth.models import User
-from django.db import DatabaseError
+from django.db import DatabaseError, IntegrityError
 from django.template.defaultfilters import slugify
 from django.utils.translation import ugettext_lazy as _
 
 from ishtar_common.unicode_csv import UnicodeWriter
 
+NEW_LINE_BREAK = '#####@@@#####'
+
 class ImportFormater(object):
     def __init__(self, field_name, formater=None, required=True, through=None,
                 through_key=None, through_dict=None, through_unicity_keys=None,
                 duplicate_field=None, regexp=None, regexp_formater_args=[],
-                reverse_for_test=None, comment=""):
+                reverse_for_test=None, force_value=None, post_processing=False,
+                concat=False, comment=""):
         self.field_name = field_name
         self.formater = formater
         self.required = required
@@ -161,6 +46,12 @@ class ImportFormater(object):
         self.regexp = regexp
         self.regexp_formater_args = regexp_formater_args
         self.reverse_for_test = reverse_for_test
+        # write this value even if a value exists
+        self.force_value = force_value
+        # post process after import
+        self.post_processing = post_processing
+        # concatenate with existing value
+        self.concat = concat
         self.comment = comment
 
     def __unicode__(self):
@@ -178,7 +69,11 @@ class ImportFormater(object):
         except TypeError:
             lst = [self.formater]
         for formater in lst:
-            formater.check(vals)
+            if formater:
+                formater.check(vals)
+
+    def post_process(self, obj, context, value, owner=None):
+        raise NotImplemented()
 
 class ImporterError(Exception):
     STANDARD = 'S'
@@ -186,6 +81,7 @@ class ImporterError(Exception):
     def __init__(self, message, type='S'):
         self.msg = message
         self.type = type
+
     def __str__(self):
         return self.msg
 
@@ -214,6 +110,7 @@ class UnicodeFormater(Formater):
                     value = value[1:]
                 if value.endswith(","):
                     value = value[:-1]
+                value = value.replace(", , ", ", ")
         except UnicodeDecodeError:
             return
         if len(value) > self.max_length:
@@ -243,19 +140,59 @@ class FloatFormater(Formater):
             raise ValueError(_(u"\"%(value)s\" is not a float") % {
                                                                  'value':value})
 
+class YearFormater(Formater):
+    def format(self, value):
+        value = value.strip()
+        if not value:
+            return
+        try:
+            value = int(value)
+            assert value > 0 and value < (datetime.date.today().year + 30)
+        except (ValueError, AssertionError):
+            raise ValueError(_(u"\"%(value)s\" is not a valid date") % {
+                                                                 'value':value})
+
+class YearNoFuturFormater(Formater):
+    def format(self, value):
+        value = value.strip()
+        if not value:
+            return
+        try:
+            value = int(value)
+            assert value > 0 and value < (datetime.date.today().year)
+        except (ValueError, AssertionError):
+            raise ValueError(_(u"\"%(value)s\" is not a valid date") % {
+                                                                 'value':value})
+
+class IntegerFormater(Formater):
+    def format(self, value):
+        value = value.strip()
+        if not value:
+            return
+        try:
+            return int(value)
+        except ValueError:
+            raise ValueError(_(u"\"%(value)s\" is not an integer") % {
+                                                                 'value':value})
+
 class StrChoiceFormater(Formater):
-    def __init__(self, choices, strict=False, equiv_dict={}, cli=False):
-        self.choices = choices
+    def __init__(self, choices, strict=False, equiv_dict={}, model=None,
+                 cli=False):
+        self.choices = list(choices)
         self.strict = strict
         self.equiv_dict = copy.deepcopy(equiv_dict)
         self.cli = cli
+        self.model = model
         self.missings = set()
         for key, value in self.choices:
             value = unicode(value)
             if not self.strict:
                 value = slugify(value)
             if value not in self.equiv_dict:
-                self.equiv_dict[value] = key
+                v = key
+                if model and v:
+                    v = model.objects.get(pk=v)
+                self.equiv_dict[value] = v
 
     def prepare(self, value):
         return unicode(value).strip()
@@ -283,7 +220,10 @@ class StrChoiceFormater(Formater):
                     pass
             res -= 1
             if res < len(self.choices):
-                self.equiv_dict[value] = self.choices[res]
+                v = self.choices[res][0]
+                if self.model and v:
+                    v = self.model.objects.get(pk=v)
+                self.equiv_dict[value] = v
             else:
                 self.equiv_dict[value] = None
 
@@ -294,11 +234,24 @@ class StrChoiceFormater(Formater):
         if value in self.equiv_dict:
             return self.equiv_dict[value]
 
+class DateFormater(Formater):
+    def __init__(self, date_format="%d/%m/%Y"):
+        self.date_format = date_format
+
+    def format(self, value):
+        value = value.strip()
+        try:
+            return datetime.datetime.strptime(value, self.date_format).date()
+        except:
+            raise ValueError(_(u"\"%(value)s\" is not a valid date") % {
+                                                           'value':value})
+
 logger = logging.getLogger(__name__)
 
 class Importer(object):
     LINE_FORMAT = []
     OBJECT_CLS = None
+    IMPORTED_LINE_FIELD = None
     UNICITY_KEYS = []
     DEFAULTS = {}
     ERRORS = {
@@ -356,23 +309,264 @@ class Importer(object):
                     vals.append([])
                 vals[idx_col].append(val)
         for idx, formater in enumerate(self.line_format):
-            formater.init(vals[idx])
+            if formater:
+                formater.init(vals[idx])
         self._initialized = True
 
     def importation(self, table):
+        self.validity_file = None
         if not self._initialized:
             self.initialize(table)
         if self.check_validity:
             with NamedTemporaryFile(delete=False) as validity_file:
-                print(validity_file.name)
-                validity_file_writer = UnicodeWriter(validity_file,
+                self.validity_file = UnicodeWriter(validity_file,
                                 delimiter=',', quotechar='"',
                                 quoting=csv.QUOTE_MINIMAL)
-                self._importation(table, validity_file_writer)
+                self._importation(table)
         else:
             self._importation(table)
 
-    def _importation(self, table, validity_file=None):
+    @classmethod
+    def _field_name_to_data_dict(cls, field_name, value, data,
+                                 force_value=False, concat=False):
+        field_names = field_name
+        if type(field_names) not in (list, tuple):
+            field_names = [field_name]
+        for field_name in field_names:
+            keys = field_name.split('__')
+            current_data = data
+            for idx, key in enumerate(keys):
+                if idx == (len(keys) - 1): # last
+                    if concat:
+                        if not value:
+                            value = ""
+                        current_data[key] = (current_data[key] + u"\n") or u""\
+                                            + value
+                    elif force_value and value:
+                        current_data[key] = value
+                    elif key not in current_data or not current_data[key]:
+                        current_data[key] = value
+                elif key not in current_data:
+                    current_data[key] = {}
+                current_data = current_data[key]
+        return data
+
+    def _importation(self, table):
+        table = list(table)
+        if not table or not table[0]:
+            raise ImporterError(self.ERRORS['no_data'], ImporterError.HEADER)
+        if self.check_col_num and len(table[0]) > len(self.line_format):
+            raise ImporterError(self.ERRORS['too_many_cols'] % {
+                     'user_col':len(table[0]), 'ref_col':len(self.line_format)})
+        self.errors = []
+        self.messages = []
+        self.number_imported = 0
+        # index of the last required column
+        for idx_last_col, formater in enumerate(reversed(self.line_format)):
+            if formater and formater.required:
+                break
+        else:
+            idx_last_col += 1
+        # min col number to be filled
+        self.min_col_number = len(self.line_format) - idx_last_col
+        # check the conformity with the reference header
+        if self.reference_header and \
+           self.skip_first_line and \
+           self.reference_header != table[0]:
+            raise ImporterError(self.ERRORS['header_check'],
+                                type=ImporterError.HEADER)
+        self.now = datetime.datetime.now()
+        for idx_line, line in enumerate(table):
+            self._line_processing(idx_line, line)
+
+    def _line_processing(self, idx_line, line):
+        if (self.skip_first_line and not idx_line):
+            if self.validity_file:
+                self.validity_file.writerow(line)
+            return
+        if not line:
+            if self.validity_file:
+                self.validity_file.writerow([])
+            return
+        self._throughs = [] # list of (formater, value)
+        self._post_processing = [] # list of (formater, value)
+        data = {}
+
+        # keep in database the raw line for testing purpose
+        if self.IMPORTED_LINE_FIELD:
+            output = io.StringIO()
+            writer = csv.writer(output)
+            writer.writerow(line)
+            data[self.IMPORTED_LINE_FIELD] = output.getvalue()
+
+        n = datetime.datetime.now()
+        logger.debug('%s - Processing line %d' % (unicode(n-self.now), idx_line))
+        self.now = n
+        n2 = n
+        self.c_errors = False
+        c_row = []
+        for idx_col, val in enumerate(line):
+            try:
+                self._row_processing(c_row, idx_col, idx_line, val, data)
+            except:
+                pass
+
+        if self.validity_file:
+            self.validity_file.writerow(c_row)
+        if not self.c_errors and (idx_col + 1) < self.min_col_number:
+            self.c_errors = True
+            self.errors.append((idx_line+1, idx_col+1,
+                          self.ERRORS['not_enough_cols'] % self.min_col_number))
+        if self.c_errors:
+            return
+        n = datetime.datetime.now()
+        logger.debug('* %s - Cols read' % (unicode(n-n2)))
+        n2 = n
+        if self.test:
+            return
+        # manage unicity of items (mainly for updates)
+        self.number_imported += 1
+        if self.UNICITY_KEYS:
+            data['defaults'] = {}
+            for k in data.keys():
+                if k not in self.UNICITY_KEYS \
+                   and k != 'defaults':
+                    data['defaults'][k] = data.pop(k)
+
+        data['history_modifier'] = self.history_modifier
+        obj, created = self.get_object(self.OBJECT_CLS, data)
+
+        if not created and 'defaults' in data:
+            for k in data['defaults']:
+                setattr(obj, k, data['defaults'][k])
+            obj.save()
+        n = datetime.datetime.now()
+        logger.debug('* %s - Item saved' % (unicode(n-n2)))
+        n2 = n
+        for formater, value in self._throughs:
+            n = datetime.datetime.now()
+            logger.debug('* %s - Processing formater %s' % (unicode(n-n2),
+                                                    formater.field_name))
+            n2 = n
+            data = {}
+            if formater.through_dict:
+                data = formater.through_dict.copy()
+            if formater.through_key:
+                data[formater.through_key] = obj
+            data[formater.field_name] = value
+            through_cls = formater.through
+            if formater.through_unicity_keys:
+                data['defaults'] = {}
+                for k in data.keys():
+                    if k not in formater.through_unicity_keys \
+                       and k != 'defaults':
+                        data['defaults'][k] = data.pop(k)
+            t_obj, created = through_cls.objects.get_or_create(**data)
+            if not created and 'defaults' in data:
+                for k in data['defaults']:
+                    setattr(t_obj, k, data['defaults'][k])
+                t_obj.save()
+
+        for formater, val in self._post_processing:
+            formater.post_process(obj, data, val, owner=self.history_modifier)
+
+    def _row_processing(self, c_row, idx_col, idx_line, val, data):
+        if idx_col >= len(self.line_format):
+            return
+
+        formater = self.line_format[idx_col]
+
+        if formater and formater.post_processing:
+            self._post_processing.append((formater, val))
+
+        if not formater or not formater.field_name:
+            if self.validity_file:
+                c_row.append(val)
+            return
+
+        # regex management
+        if formater.regexp:
+            # multiline regexp is a mess...
+            val = val.replace('\n', NEW_LINE_BREAK)
+            match = formater.regexp.match(val)
+            if not match:
+                if formater.required:
+                    self.errors.append((idx_line+1, idx_col+1,
+                                       self.ERRORS['value_required']))
+                elif not val.strip():
+                    c_row.append("")
+                    return
+                self.c_errors = True
+                val = val.replace(NEW_LINE_BREAK, '\n')
+                self.errors.append((idx_line+1, idx_col+1,
+                         unicode(self.ERRORS['regex_not_match']) + val))
+                c_row.append("")
+                return
+            val_group = [v.replace(NEW_LINE_BREAK, '\n')
+                         for v in match.groups()]
+        else:
+            val_group = [val]
+
+        c_values = []
+        for idx_v, v in enumerate(val_group):
+            self.message = ''
+            func = formater.formater
+            if type(func) in (list, tuple):
+                func = func[idx_v]
+            if not callable(func) and type(func) in (unicode, str):
+                func = getattr(self, func)
+            value = None
+
+            try:
+                if formater.regexp_formater_args:
+                    args = []
+                    for idx in formater.regexp_formater_args[idx_v]:
+                        args.append(val_group[idx])
+                    value = func.format(*args)
+                else:
+                    value = func.format(v)
+            except ValueError, e:
+                if formater.required:
+                    self.c_errors = True
+                self.errors.append((idx_line+1, idx_col+1, e.message))
+                c_values.append(None)
+                return
+
+            if self.message:
+                self.messages.append(self.message)
+            c_values.append(value)
+
+            if value == None:
+                if formater.required:
+                    self.c_errors = True
+                    self.errors.append((idx_line+1, idx_col+1,
+                                       self.ERRORS['value_required']))
+                return
+
+            field_name = formater.field_name
+            if type(field_name) in (list, tuple):
+                field_name = field_name[idx_v]
+            field_names = [field_name]
+            if formater.duplicate_field:
+                duplicate_field = formater.duplicate_field
+                if type(duplicate_field) in (list, tuple):
+                    duplicate_field = duplicate_field[idx_v]
+                field_names += [duplicate_field]
+
+            if formater.through:
+                self._throughs.append((formater, value))
+            else:
+                for field_name in field_names:
+                    self._field_name_to_data_dict(field_name,
+                                              value, data, formater.force_value)
+        if formater.reverse_for_test:
+            c_row.append(formater.reverse_for_test(**c_values))
+        else:
+            c_row.append(unicode(c_values))
+
+
+    """
+    def _importation(self, table):
         table = list(table)
         if not table or not table[0]:
             raise ImporterError(self.ERRORS['no_data'], ImporterError.HEADER)
@@ -398,33 +592,46 @@ class Importer(object):
                                 type=ImporterError.HEADER)
         now = datetime.datetime.now()
         for idx_line, line in enumerate(table):
+            #self._line_processing()
+
             if (self.skip_first_line and not idx_line):
-                if validity_file:
-                    validity_file.writerow(line)
+                if self.validity_file:
+                    self.validity_file.writerow(line)
                 continue
             if not line:
-                if validity_file:
-                    validity_file.writerow([])
+                if self.validity_file:
+                    self.validity_file.writerow([])
                 continue
-            throughs = [] # list of (formater, value)
+            self.throughs = [] # list of (formater, value)
+            self.post_processing = [] # list of (formater, value)
             data = {}
+
+            # keep in database the raw line for testing purpose
+            if self.IMPORTED_LINE_FIELD:
+                output = io.StringIO()
+                writer = csv.writer(output)
+                writer.writerow(line)
+                data[self.IMPORTED_LINE_FIELD] = output.getvalue()
+
             n = datetime.datetime.now()
             logger.debug('%s - Processing line %d' % (unicode(n-now), idx_line))
             now = n
             n2 = n
-            c_errors = False
+            self.c_errors = False
             c_row = []
             for idx_col, val in enumerate(line):
+                #self._row_processing(self, c_row, idx_col, val):
+
                 if idx_col >= len(self.line_format):
                     break
                 formater = self.line_format[idx_col]
                 if not formater.field_name:
-                    if validity_file:
+                    if self.validity_file:
                         c_row.append(val)
                     continue
                 if formater.regexp:
                     # multiline regexp is a mess...
-                    val = val.replace('\n', '######???#####')
+                    val = val.replace('\n', NEW_LINE_BREAK)
                     match = formater.regexp.match(val)
                     if not match:
                         if formater.required:
@@ -434,12 +641,12 @@ class Importer(object):
                             c_row.append("")
                             continue
                         c_errors = True
-                        val = val.replace('######???#####', '\n')
+                        val = val.replace(NEW_LINE_BREAK, '\n')
                         self.errors.append((idx_line+1, idx_col+1,
                                  unicode(self.ERRORS['regex_not_match']) + val))
                         c_row.append("")
                         continue
-                    val_group = [v.replace('######???#####', '\n')
+                    val_group = [v.replace(NEW_LINE_BREAK, '\n')
                                  for v in match.groups()]
                 else:
                     val_group = [val]
@@ -483,29 +690,26 @@ class Importer(object):
                         if type(duplicate_field) in (list, tuple):
                             duplicate_field = duplicate_field[idx_v]
                         field_names += [duplicate_field]
-                    if not formater.through:
-                        for field_name in field_names:
-                            keys = field_name.split('__')
-                            current_data = data
-                            for idx, key in enumerate(keys):
-                                if idx == (len(keys) - 1): # last
-                                    current_data[key] = value
-                                elif key not in current_data:
-                                    current_data[key] = {}
-                                current_data = current_data[key]
-                    else:
+
+
+                    if formater.through:
                         throughs.append((formater, value))
+                    else:
+                        for field_name in field_names:
+                            self._field_name_to_data_dict(field_name,
+                                                          value, data)
                 if formater.reverse_for_test:
                     c_row.append(formater.reverse_for_test(**c_values))
                 else:
                     c_row.append(unicode(c_values))
-            if validity_file:
-                validity_file.writerow(c_row)
-            if not c_errors and (idx_col + 1) < min_col_number:
-                c_errors = True
+
+            if self.validity_file:
+                self.validity_file.writerow(c_row)
+            if not self.c_errors and (idx_col + 1) < min_col_number:
+                self.c_errors = True
                 self.errors.append((idx_line+1, idx_col+1,
                                self.ERRORS['not_enough_cols'] % min_col_number))
-            if c_errors:
+            if self.c_errors:
                 continue
             n = datetime.datetime.now()
             logger.debug('* %s - Cols read' % (unicode(n-n2)))
@@ -530,7 +734,7 @@ class Importer(object):
             n = datetime.datetime.now()
             logger.debug('* %s - Item saved' % (unicode(n-n2)))
             n2 = n
-            for formater, value in throughs:
+            for formater, value in self.throughs:
                 n = datetime.datetime.now()
                 logger.debug('* %s - Processing formater %s' % (unicode(n-n2),
                                                         formater.field_name))
@@ -553,6 +757,7 @@ class Importer(object):
                     for k in data['defaults']:
                         setattr(t_obj, k, data['defaults'][k])
                     t_obj.save()
+    """
 
     def get_object(self, cls, data, path=[]):
         m2ms = []
@@ -563,7 +768,8 @@ class Importer(object):
                     continue
                 field_object, model, direct, m2m = \
                                     cls._meta.get_field_by_name(attribute)
-                if field_object.rel and type(data[attribute]) == dict:
+                if hasattr(field_object, 'rel') and field_object.rel and \
+                   type(data[attribute]) == dict:
                     c_path.append(attribute)
                     # put history_modifier for every created item
                     data[attribute]['history_modifier'] = \
@@ -577,15 +783,26 @@ class Importer(object):
                 for k in self._defaults[path]:
                     if k not in data or not data[k]:
                         data[k] = self._defaults[path][k]
-            obj, created = cls.objects.get_or_create(**data)
-            for attr, value in m2ms:
-                getattr(obj, attr).add(value)
+
+            # filter default values
+            create_dict = copy.deepcopy(data)
+            for k in create_dict.keys():
+                if type(create_dict[k]) == dict:
+                    create_dict.pop(k)
+
+            try:
+                obj, created = cls.objects.get_or_create(**create_dict)
+                for attr, value in m2ms:
+                    getattr(obj, attr).add(value)
+            except IntegrityError:
+                raise ImporterError("Erreur d'import %s, contexte : %s" \
+                                % (unicode(cls), unicode(data)))
             return obj, created
         return data
 
     def get_csv_errors(self):
         for line, col, error in self.errors:
-            print '"%d","%d","%s"' % (line, col, unicode(error))
+            print('"%d","%d","%s"' % (line, col, unicode(error)))
 
     @classmethod
     def choices_check(cls, choices):