Improve/fix import management. Bibracte import for operations.

2 files changed, 118 insertions, 29 deletions

diff --git a/archaeological_operations/management/commands/import_operations.py b/archaeological_operations/management/commands/import_operations.py
index a9ecf41c9..fe4afc032 100755..100644
--- a/archaeological_operations/management/commands/import_operations.py
+++ b/archaeological_operations/management/commands/import_operations.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright (C) 2012-2013  Étienne Loks <etienne.loks_AT_peacefrogsDOTnet>
+# Copyright (C) 2015  Étienne Loks <etienne.loks_AT_peacefrogsDOTnet>
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -17,40 +17,73 @@
 
 # See the file COPYING for details.
 
+import datetime, unicodecsv
+
+from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
-from archaeological_operations.import_from_csv import import_from_csv
-from archaeological_operations.import_from_dbf import import_from_dbf
 
-IMPORTERS = {'csv':import_from_csv,
-             'dbf':import_from_dbf,
-             'db3':import_from_dbf,
-             'fp':import_from_dbf,
-             'vfp':import_from_dbf}
+from archaeological_operations.data_importer import *
+
+IMPORTERS = {'bibracte-operation':OperationImporterBibracte}
 
 class Command(BaseCommand):
-    args = '<filename> [<lines>]'
-    help = "Import archaelogical operations"
+    args = '<filename> <importer_name> [<nb lines skipped>]'
+    help = "Import archaeological operations"
 
     def handle(self, *args, **options):
         if not args or not args[0]:
             raise CommandError("No file provided.")
+        if len(args) < 2 or args[1] not in IMPORTERS:
+            msg = "Bad importer. \nAvailable importers are:\n"
+            for key in IMPORTERS:
+                msg += "\t* %s: %s" % (key, IMPORTERS[key].DESC.encode('utf-8')
+                                     or "-")
+            raise CommandError(msg)
+        try:
+            skip_lines = int(args[2])
+        except:
+            skip_lines = 0
         filename = args[0]
-        update = True
-        file_type = None
-        lines = len(args) > 1 and args[1]
-        if not file_type:
-            suffix = filename.split('.')[-1].lower()
-            if suffix in IMPORTERS.keys():
-                file_type = suffix
-            else:
-                raise CommandError("This file extension is not managed. "\
-                                    "Specify manualy the file type.")
-        elif file_type not in IMPORTERS.keys():
-            raise CommandError("This file type is not managed.")
-        nb_ops, errors = IMPORTERS[file_type](filename,
-                                              update=update,
-                                              stdout=self.stdout,
-                                              lines=lines)
-        self.stdout.write('\n* %d operation treated\n' % nb_ops)
-        if errors:
-            self.stderr.write('\n'.join(errors))
+        importer = IMPORTERS[args[1]](skip_lines=skip_lines, output='cli')
+        sys.stdout.write("*" * 72 + "\n")
+        msg = "* Importer - %s" % importer.DESC
+        if len(msg) < 72:
+            msg += (71 - len(msg))*" " + "*\n"
+        sys.stdout.write(msg)
+        sys.stdout.write("*" * 72 + "\n\n")
+        sys.stdout.write("Processing...")
+        with open(filename) as csv_file:
+            encodings = [settings.ENCODING, settings.ALT_ENCODING, 'utf-8']
+            for encoding in encodings:
+                try:
+                    importer.importation([line for line in
+                            unicodecsv.reader(csv_file, encoding='utf-8')])
+                    errors = importer.get_csv_errors()
+                    sys.stdout.write("\n")
+                    if errors:
+                        print errors
+                        now = datetime.datetime.now().isoformat('-'
+                                                ).replace(':','')
+                        error_file = '.'.join(filename.split('.')[:-1]) \
+                                     + "_errors_%s.csv" % now
+                        sys.stdout.write("Some errors as occured during the ")
+                        sys.stdout.write("import.\n")
+                        try:
+                            with open(error_file, 'w') as fle:
+                                fle.write(errors.encode('utf-8'))
+                            sys.stdout.write("A report has been create in file:"\
+                                             " \"%s\"" % error_file)
+                        except IOError:
+                            sys.stdout.write("Cannot create CSV error file \"%s\"." %
+                                             error_file)
+                    break
+                except ImporterError, e:
+                    if e.type == ImporterError.HEADER and encoding != encodings[-1]:
+                        csv_file.seek(0)
+                        continue
+                except UnicodeDecodeError:
+                    if encoding != encodings[-1]:
+                        csv_file.seek(0)
+                        continue
+        sys.stdout.write("\n\n")
+
diff --git a/archaeological_operations/management/commands/import_operations_old.py b/archaeological_operations/management/commands/import_operations_old.py
new file mode 100755
index 000000000..a9ecf41c9
--- /dev/null
+++ b/archaeological_operations/management/commands/import_operations_old.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2012-2013  Étienne Loks <etienne.loks_AT_peacefrogsDOTnet>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# See the file COPYING for details.
+
+from django.core.management.base import BaseCommand, CommandError
+from archaeological_operations.import_from_csv import import_from_csv
+from archaeological_operations.import_from_dbf import import_from_dbf
+
+IMPORTERS = {'csv':import_from_csv,
+             'dbf':import_from_dbf,
+             'db3':import_from_dbf,
+             'fp':import_from_dbf,
+             'vfp':import_from_dbf}
+
+class Command(BaseCommand):
+    args = '<filename> [<lines>]'
+    help = "Import archaelogical operations"
+
+    def handle(self, *args, **options):
+        if not args or not args[0]:
+            raise CommandError("No file provided.")
+        filename = args[0]
+        update = True
+        file_type = None
+        lines = len(args) > 1 and args[1]
+        if not file_type:
+            suffix = filename.split('.')[-1].lower()
+            if suffix in IMPORTERS.keys():
+                file_type = suffix
+            else:
+                raise CommandError("This file extension is not managed. "\
+                                    "Specify manualy the file type.")
+        elif file_type not in IMPORTERS.keys():
+            raise CommandError("This file type is not managed.")
+        nb_ops, errors = IMPORTERS[file_type](filename,
+                                              update=update,
+                                              stdout=self.stdout,
+                                              lines=lines)
+        self.stdout.write('\n* %d operation treated\n' % nb_ops)
+        if errors:
+            self.stderr.write('\n'.join(errors))