Improve/fix import management. Bibracte import for operations.

5 files changed, 151 insertions, 250 deletions

diff --git a/archaeological_operations/data_importer.py b/archaeological_operations/data_importer.py
index e02b53df0..ebcc039d0 100644
--- a/archaeological_operations/data_importer.py
+++ b/archaeological_operations/data_importer.py
@@ -24,9 +24,9 @@ from django.template.defaultfilters import slugify
 from django.utils.translation import ugettext_lazy as _
 
 from ishtar_common.data_importer import *
-from ishtar_common.models import Town
+from ishtar_common.models import Town, OrganizationType
 
-from archaeological_operations.models import Parcel
+from archaeological_operations import models
 from archaeological_operations.utils import parse_parcels
 
 RE_PERMIT_REFERENCE = re.compile('[A-Za-z]*(.*)')
@@ -45,7 +45,7 @@ class ImportParcelFormater(ImportFormater):
         for parcel_dct in parse_parcels(value, owner=owner):
             parcel_dct.update(base_dct)
             try:
-                Parcel.objects.get_or_create(**parcel_dct)
+                models.Parcel.objects.get_or_create(**parcel_dct)
             except IntegrityError:
                 raise ImporterError("Erreur d'import parcelle, contexte : %s" \
                                                     % unicode(parcel_dct))
@@ -148,8 +148,9 @@ RE_ORGA = re.compile("([^,]*)")
 
 
 class OperationImporterBibracte(Importer):
+    DESC = u"Exports Bibracte : importeur pour l'onglet opération"
     DEFAULTS = {
-                ('in_charge',):{
+                ('operator',):{
                     'organization_type':OrganizationType.objects.get(
                                              txt_idx="operator")},
                }
@@ -171,10 +172,8 @@ class OperationImporterBibracte(Importer):
         # fin
         ImportFormater('excavation_end_date', DateFormater('%Y/%m/%d'),),
         # Chronos
-        ImportFormater('periods', TypeFormater(models.Period, many=True),
-                       many=True),
+        ImportFormater('periods', TypeFormater(models.Period, many_split="&")),
     ]
 
     OBJECT_CLS = models.Operation
-    DEFAULTS = {}
 
diff --git a/archaeological_operations/management/commands/import_operations.py b/archaeological_operations/management/commands/import_operations.py
index a9ecf41c9..fe4afc032 100755..100644
--- a/archaeological_operations/management/commands/import_operations.py
+++ b/archaeological_operations/management/commands/import_operations.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright (C) 2012-2013  Étienne Loks <etienne.loks_AT_peacefrogsDOTnet>
+# Copyright (C) 2015  Étienne Loks <etienne.loks_AT_peacefrogsDOTnet>
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -17,40 +17,73 @@
 
 # See the file COPYING for details.
 
+import datetime, unicodecsv
+
+from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
-from archaeological_operations.import_from_csv import import_from_csv
-from archaeological_operations.import_from_dbf import import_from_dbf
 
-IMPORTERS = {'csv':import_from_csv,
-             'dbf':import_from_dbf,
-             'db3':import_from_dbf,
-             'fp':import_from_dbf,
-             'vfp':import_from_dbf}
+from archaeological_operations.data_importer import *
+
+IMPORTERS = {'bibracte-operation':OperationImporterBibracte}
 
 class Command(BaseCommand):
-    args = '<filename> [<lines>]'
-    help = "Import archaelogical operations"
+    args = '<filename> <importer_name> [<nb lines skipped>]'
+    help = "Import archaeological operations"
 
     def handle(self, *args, **options):
         if not args or not args[0]:
             raise CommandError("No file provided.")
+        if len(args) < 2 or args[1] not in IMPORTERS:
+            msg = "Bad importer. \nAvailable importers are:\n"
+            for key in IMPORTERS:
+                msg += "\t* %s: %s" % (key, IMPORTERS[key].DESC.encode('utf-8')
+                                     or "-")
+            raise CommandError(msg)
+        try:
+            skip_lines = int(args[2])
+        except:
+            skip_lines = 0
         filename = args[0]
-        update = True
-        file_type = None
-        lines = len(args) > 1 and args[1]
-        if not file_type:
-            suffix = filename.split('.')[-1].lower()
-            if suffix in IMPORTERS.keys():
-                file_type = suffix
-            else:
-                raise CommandError("This file extension is not managed. "\
-                                    "Specify manualy the file type.")
-        elif file_type not in IMPORTERS.keys():
-            raise CommandError("This file type is not managed.")
-        nb_ops, errors = IMPORTERS[file_type](filename,
-                                              update=update,
-                                              stdout=self.stdout,
-                                              lines=lines)
-        self.stdout.write('\n* %d operation treated\n' % nb_ops)
-        if errors:
-            self.stderr.write('\n'.join(errors))
+        importer = IMPORTERS[args[1]](skip_lines=skip_lines, output='cli')
+        sys.stdout.write("*" * 72 + "\n")
+        msg = "* Importer - %s" % importer.DESC
+        if len(msg) < 72:
+            msg += (71 - len(msg))*" " + "*\n"
+        sys.stdout.write(msg)
+        sys.stdout.write("*" * 72 + "\n\n")
+        sys.stdout.write("Processing...")
+        with open(filename) as csv_file:
+            encodings = [settings.ENCODING, settings.ALT_ENCODING, 'utf-8']
+            for encoding in encodings:
+                try:
+                    importer.importation([line for line in
+                            unicodecsv.reader(csv_file, encoding='utf-8')])
+                    errors = importer.get_csv_errors()
+                    sys.stdout.write("\n")
+                    if errors:
+                        print errors
+                        now = datetime.datetime.now().isoformat('-'
+                                                ).replace(':','')
+                        error_file = '.'.join(filename.split('.')[:-1]) \
+                                     + "_errors_%s.csv" % now
+                        sys.stdout.write("Some errors as occured during the ")
+                        sys.stdout.write("import.\n")
+                        try:
+                            with open(error_file, 'w') as fle:
+                                fle.write(errors.encode('utf-8'))
+                            sys.stdout.write("A report has been create in file:"\
+                                             " \"%s\"" % error_file)
+                        except IOError:
+                            sys.stdout.write("Cannot create CSV error file \"%s\"." %
+                                             error_file)
+                    break
+                except ImporterError, e:
+                    if e.type == ImporterError.HEADER and encoding != encodings[-1]:
+                        csv_file.seek(0)
+                        continue
+                except UnicodeDecodeError:
+                    if encoding != encodings[-1]:
+                        csv_file.seek(0)
+                        continue
+        sys.stdout.write("\n\n")
+
diff --git a/archaeological_operations/management/commands/import_operations_old.py b/archaeological_operations/management/commands/import_operations_old.py
new file mode 100755
index 000000000..a9ecf41c9
--- /dev/null
+++ b/archaeological_operations/management/commands/import_operations_old.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2012-2013  Étienne Loks <etienne.loks_AT_peacefrogsDOTnet>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# See the file COPYING for details.
+
+from django.core.management.base import BaseCommand, CommandError
+from archaeological_operations.import_from_csv import import_from_csv
+from archaeological_operations.import_from_dbf import import_from_dbf
+
+IMPORTERS = {'csv':import_from_csv,
+             'dbf':import_from_dbf,
+             'db3':import_from_dbf,
+             'fp':import_from_dbf,
+             'vfp':import_from_dbf}
+
+class Command(BaseCommand):
+    args = '<filename> [<lines>]'
+    help = "Import archaelogical operations"
+
+    def handle(self, *args, **options):
+        if not args or not args[0]:
+            raise CommandError("No file provided.")
+        filename = args[0]
+        update = True
+        file_type = None
+        lines = len(args) > 1 and args[1]
+        if not file_type:
+            suffix = filename.split('.')[-1].lower()
+            if suffix in IMPORTERS.keys():
+                file_type = suffix
+            else:
+                raise CommandError("This file extension is not managed. "\
+                                    "Specify manualy the file type.")
+        elif file_type not in IMPORTERS.keys():
+            raise CommandError("This file type is not managed.")
+        nb_ops, errors = IMPORTERS[file_type](filename,
+                                              update=update,
+                                              stdout=self.stdout,
+                                              lines=lines)
+        self.stdout.write('\n* %d operation treated\n' % nb_ops)
+        if errors:
+            self.stderr.write('\n'.join(errors))
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index 0ce61ba01..87b3a40b0 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -17,7 +17,7 @@
 
 # See the file COPYING for details.
 
-import copy, csv, datetime, logging, sys
+import copy, csv, datetime, logging, re, sys
 from tempfile import NamedTemporaryFile
 
 from django.contrib.auth.models import User
@@ -216,16 +216,16 @@ class StrChoiceFormater(Formater):
         msgstr += unicode(_(u"%d. None of the above - skip")) % idx + u"\n"
         if self.many_split:
             new_values = []
-            r = re.compile(func.many_split)
+            r = re.compile(self.many_split)
             for value in values:
                 new_values += r.split(value)
             values = new_values
         for value in values:
-            base_value = copy(value)
+            base_value = copy.copy(value)
             value = self.prepare(value)
             if value in self.equiv_dict:
                 continue
-            if not self.cli:
+            if output != 'cli':
                 self.missings.add(value)
                 continue
             res = None
@@ -263,9 +263,12 @@ class StrChoiceFormater(Formater):
 
 class TypeFormater(StrChoiceFormater):
     def __init__(self, model, cli=False, defaults={}, many_split=False):
+        self.create = True
+        self.strict = False
         self.model = model
         self.defaults = defaults
         self.many_split = many_split
+        self.missings = set()
         self.equiv_dict, self.choices = {}, []
         for item in model.objects.all():
             self.choices.append((item.pk, unicode(item)))
@@ -348,6 +351,7 @@ class StrToBoolean(Formater):
 logger = logging.getLogger(__name__)
 
 class Importer(object):
+    DESC = ""
     LINE_FORMAT = []
     OBJECT_CLS = None
     IMPORTED_LINE_FIELD = None
@@ -366,19 +370,18 @@ class Importer(object):
         'regex_not_match':_(u"The regexp doesn't match.")
         }
 
-    def __init__(self, skip_first_line=False, reference_header=None,
+    def __init__(self, skip_lines=0, reference_header=None,
                  check_col_num=False, test=False, check_validity=True,
-                 history_modifier=None, output=None):
+                 history_modifier=None, output='silent'):
         """
-         * skip_first_line must be set to True if the data provided has got
-           an header.
+         * skip_line must be set if the data provided has got headers lines.
          * a reference_header can be provided to perform a data compliance
            check. It can be useful to warn about bad parsing.
          * test doesn't write in the database
          * check_validity rewrite a CSV file to be compared
         """
         self.message = ''
-        self.skip_first_line = skip_first_line
+        self.skip_lines = skip_lines
         self.reference_header = reference_header
         self.test = test
         self.errors = [] # list of (line, col, message)
@@ -408,7 +411,7 @@ class Importer(object):
         assert output in ('silent', 'cli', 'db')
         vals = []
         for idx_line, line in enumerate(table):
-            if (self.skip_first_line and not idx_line):
+            if self.skip_lines > idx_line:
                 continue
             for idx_col, val in enumerate(line):
                 if idx_col >= len(self.line_format):
@@ -424,7 +427,7 @@ class Importer(object):
     def importation(self, table):
         self.validity_file = None
         if not self._initialized:
-            self.initialize(table)
+            self.initialize(table, self.output)
         if self.check_validity:
             with NamedTemporaryFile(delete=False) as validity_file:
                 self.validity_file = UnicodeWriter(validity_file,
@@ -479,7 +482,7 @@ class Importer(object):
         self.min_col_number = len(self.line_format) - idx_last_col
         # check the conformity with the reference header
         if self.reference_header and \
-           self.skip_first_line and \
+           self.skip_lines and \
            self.reference_header != table[0]:
             raise ImporterError(self.ERRORS['header_check'],
                                 type=ImporterError.HEADER)
@@ -507,7 +510,7 @@ class Importer(object):
                 self.errors.append((idx_line, None, msg))
 
     def _line_processing(self, idx_line, line):
-        if (self.skip_first_line and not idx_line):
+        if self.skip_lines > idx_line:
             if self.validity_file:
                 self.validity_file.writerow(line)
             return
@@ -703,201 +706,6 @@ class Importer(object):
         else:
             c_row.append(unicode(c_values))
 
-
-    """
-    def _importation(self, table):
-        table = list(table)
-        if not table or not table[0]:
-            raise ImporterError(self.ERRORS['no_data'], ImporterError.HEADER)
-        if self.check_col_num and len(table[0]) > len(self.line_format):
-            raise ImporterError(self.ERRORS['too_many_cols'] % {
-                     'user_col':len(table[0]), 'ref_col':len(self.line_format)})
-        self.errors = []
-        self.messages = []
-        self.number_imported = 0
-        # index of the last required column
-        for idx_last_col, formater in enumerate(reversed(self.line_format)):
-            if formater.required:
-                break
-        else:
-            idx_last_col += 1
-        # min col number to be filled
-        min_col_number = len(self.line_format) - idx_last_col
-        # check the conformity with the reference header
-        if self.reference_header and \
-           self.skip_first_line and \
-           self.reference_header != table[0]:
-            raise ImporterError(self.ERRORS['header_check'],
-                                type=ImporterError.HEADER)
-        now = datetime.datetime.now()
-        for idx_line, line in enumerate(table):
-            #self._line_processing()
-
-            if (self.skip_first_line and not idx_line):
-                if self.validity_file:
-                    self.validity_file.writerow(line)
-                continue
-            if not line:
-                if self.validity_file:
-                    self.validity_file.writerow([])
-                continue
-            self.throughs = [] # list of (formater, value)
-            self.post_processing = [] # list of (formater, value)
-            data = {}
-
-            # keep in database the raw line for testing purpose
-            if self.IMPORTED_LINE_FIELD:
-                output = io.StringIO()
-                writer = csv.writer(output)
-                writer.writerow(line)
-                data[self.IMPORTED_LINE_FIELD] = output.getvalue()
-
-            n = datetime.datetime.now()
-            logger.debug('%s - Processing line %d' % (unicode(n-now), idx_line))
-            now = n
-            n2 = n
-            self.c_errors = False
-            c_row = []
-            for idx_col, val in enumerate(line):
-                #self._row_processing(self, c_row, idx_col, val):
-
-                if idx_col >= len(self.line_format):
-                    break
-                formater = self.line_format[idx_col]
-                if not formater.field_name:
-                    if self.validity_file:
-                        c_row.append(val)
-                    continue
-                if formater.regexp:
-                    # multiline regexp is a mess...
-                    val = val.replace('\n', NEW_LINE_BREAK)
-                    match = formater.regexp.match(val)
-                    if not match:
-                        if formater.required:
-                            self.errors.append((idx_line+1, idx_col+1,
-                                               self.ERRORS['value_required']))
-                        elif not val.strip():
-                            c_row.append("")
-                            continue
-                        c_errors = True
-                        val = val.replace(NEW_LINE_BREAK, '\n')
-                        self.errors.append((idx_line+1, idx_col+1,
-                                 unicode(self.ERRORS['regex_not_match']) + val))
-                        c_row.append("")
-                        continue
-                    val_group = [v.replace(NEW_LINE_BREAK, '\n')
-                                 for v in match.groups()]
-                else:
-                    val_group = [val]
-                c_values = []
-                for idx_v, v in enumerate(val_group):
-                    self.message = ''
-                    func = formater.formater
-                    if type(func) in (list, tuple):
-                        func = func[idx_v]
-                    if not callable(func) and type(func) in (unicode, str):
-                        func = getattr(self, func)
-                    value = None
-                    try:
-                        if formater.regexp_formater_args:
-                            args = []
-                            for idx in formater.regexp_formater_args[idx_v]:
-                                args.append(val_group[idx])
-                            value = func.format(*args)
-                        else:
-                            value = func.format(v)
-                    except ValueError, e:
-                        c_errors = True
-                        self.errors.append((idx_line+1, idx_col+1, e.message))
-                        c_values.append(None)
-                        continue
-                    if self.message:
-                        self.messages.append(self.message)
-                    c_values.append(value)
-                    if value == None:
-                        if formater.required:
-                            c_errors = True
-                            self.errors.append((idx_line+1, idx_col+1,
-                                               self.ERRORS['value_required']))
-                        continue
-                    field_name = formater.field_name
-                    if type(field_name) in (list, tuple):
-                        field_name = field_name[idx_v]
-                    field_names = [field_name]
-                    if formater.duplicate_field:
-                        duplicate_field = formater.duplicate_field
-                        if type(duplicate_field) in (list, tuple):
-                            duplicate_field = duplicate_field[idx_v]
-                        field_names += [duplicate_field]
-
-
-                    if formater.through:
-                        throughs.append((formater, value))
-                    else:
-                        for field_name in field_names:
-                            self._field_name_to_data_dict(field_name,
-                                                          value, data)
-                if formater.reverse_for_test:
-                    c_row.append(formater.reverse_for_test(**c_values))
-                else:
-                    c_row.append(unicode(c_values))
-
-            if self.validity_file:
-                self.validity_file.writerow(c_row)
-            if not self.c_errors and (idx_col + 1) < min_col_number:
-                self.c_errors = True
-                self.errors.append((idx_line+1, idx_col+1,
-                               self.ERRORS['not_enough_cols'] % min_col_number))
-            if self.c_errors:
-                continue
-            n = datetime.datetime.now()
-            logger.debug('* %s - Cols read' % (unicode(n-n2)))
-            n2 = n
-            if self.test:
-                continue
-            # manage unicity of items (mainly for updates)
-            self.number_imported += 1
-            if self.UNICITY_KEYS:
-                data['defaults'] = {}
-                for k in data.keys():
-                    if k not in self.UNICITY_KEYS \
-                       and k != 'defaults':
-                        data['defaults'][k] = data.pop(k)
-
-            obj, created = self.get_object(self.OBJECT_CLS, data)
-
-            if not created and 'defaults' in data:
-                for k in data['defaults']:
-                    setattr(obj, k, data['defaults'][k])
-                obj.save()
-            n = datetime.datetime.now()
-            logger.debug('* %s - Item saved' % (unicode(n-n2)))
-            n2 = n
-            for formater, value in self.throughs:
-                n = datetime.datetime.now()
-                logger.debug('* %s - Processing formater %s' % (unicode(n-n2),
-                                                        formater.field_name))
-                n2 = n
-                data = {}
-                if formater.through_dict:
-                    data = formater.through_dict.copy()
-                if formater.through_key:
-                    data[formater.through_key] = obj
-                data[formater.field_name] = value
-                through_cls = formater.through
-                if formater.through_unicity_keys:
-                    data['defaults'] = {}
-                    for k in data.keys():
-                        if k not in formater.through_unicity_keys \
-                           and k != 'defaults':
-                            data['defaults'][k] = data.pop(k)
-                t_obj, created = through_cls.objects.get_or_create(**data)
-                if not created and 'defaults' in data:
-                    for k in data['defaults']:
-                        setattr(t_obj, k, data['defaults'][k])
-                    t_obj.save()
-    """
-
     def get_object(self, cls, data, path=[]):
         m2ms = []
         if data and type(data) == dict:
@@ -937,6 +745,8 @@ class Importer(object):
                     dct = create_dict.copy()
                     dct['defaults'] = defaults
                     obj, created = cls.objects.get_or_create(**create_dict)
+                except IntegrityError as e:
+                    raise IntegrityError(e.message)
                 except:
                     created = False
                     obj = cls.objects.filter(**create_dict).all()[0]
@@ -946,16 +756,19 @@ class Importer(object):
                         values = value
                     for v in values:
                         getattr(obj, attr).add(v)
-            except IntegrityError:
-                raise ImporterError("Erreur d'import %s, contexte : %s" \
-                                % (unicode(cls), unicode(data)))
+            except IntegrityError as e:
+                raise ImporterError("Erreur d'import %s, contexte : %s, erreur : %s" \
+                        % (unicode(cls), unicode(data), e.message.decode('utf-8')))
             return obj, created
         return data
 
     def get_csv_errors(self):
-        csv_errors = []
+        if not self.errors:
+            return ""
+        csv_errors = ["line,col,error"]
         for line, col, error in self.errors:
-            csv_errors.append(u'"%d","%d","%s"' % (line or 0, col or 0,
+            csv_errors.append(u'"%s","%s","%s"' % (line and unicode(line) or '-',
+                                                   col and unicode(col) or '-',
                                                         unicode(error)))
         return u"\n".join(csv_errors)
 
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index 7ef06ddb7..280376966 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -386,7 +386,7 @@ class GeneralType(models.Model):
         keys = []
         content_type = ContentType.objects.get_for_model(self.__class__)
         for ik in ItemKey.objects.filter(content_type=content_type,
-                                         object_id=ik.pk).all():
+                                         object_id=self.pk).all():
             keys.append(ik.key)
         return keys