Improve/fix import management. Bibracte import for operations.

1 files changed, 62 insertions, 29 deletions

diff --git a/archaeological_operations/management/commands/import_operations.py b/archaeological_operations/management/commands/import_operations.py
index a9ecf41c9..fe4afc032 100755..100644
--- a/archaeological_operations/management/commands/import_operations.py
+++ b/archaeological_operations/management/commands/import_operations.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright (C) 2012-2013  Étienne Loks <etienne.loks_AT_peacefrogsDOTnet>
+# Copyright (C) 2015  Étienne Loks <etienne.loks_AT_peacefrogsDOTnet>
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -17,40 +17,73 @@
 
 # See the file COPYING for details.
 
+import datetime, unicodecsv
+
+from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
-from archaeological_operations.import_from_csv import import_from_csv
-from archaeological_operations.import_from_dbf import import_from_dbf
 
-IMPORTERS = {'csv':import_from_csv,
-             'dbf':import_from_dbf,
-             'db3':import_from_dbf,
-             'fp':import_from_dbf,
-             'vfp':import_from_dbf}
+from archaeological_operations.data_importer import *
+
+IMPORTERS = {'bibracte-operation':OperationImporterBibracte}
 
 class Command(BaseCommand):
-    args = '<filename> [<lines>]'
-    help = "Import archaelogical operations"
+    args = '<filename> <importer_name> [<nb lines skipped>]'
+    help = "Import archaeological operations"
 
     def handle(self, *args, **options):
         if not args or not args[0]:
             raise CommandError("No file provided.")
+        if len(args) < 2 or args[1] not in IMPORTERS:
+            msg = "Bad importer. \nAvailable importers are:\n"
+            for key in IMPORTERS:
+                msg += "\t* %s: %s" % (key, IMPORTERS[key].DESC.encode('utf-8')
+                                     or "-")
+            raise CommandError(msg)
+        try:
+            skip_lines = int(args[2])
+        except:
+            skip_lines = 0
         filename = args[0]
-        update = True
-        file_type = None
-        lines = len(args) > 1 and args[1]
-        if not file_type:
-            suffix = filename.split('.')[-1].lower()
-            if suffix in IMPORTERS.keys():
-                file_type = suffix
-            else:
-                raise CommandError("This file extension is not managed. "\
-                                    "Specify manualy the file type.")
-        elif file_type not in IMPORTERS.keys():
-            raise CommandError("This file type is not managed.")
-        nb_ops, errors = IMPORTERS[file_type](filename,
-                                              update=update,
-                                              stdout=self.stdout,
-                                              lines=lines)
-        self.stdout.write('\n* %d operation treated\n' % nb_ops)
-        if errors:
-            self.stderr.write('\n'.join(errors))
+        importer = IMPORTERS[args[1]](skip_lines=skip_lines, output='cli')
+        sys.stdout.write("*" * 72 + "\n")
+        msg = "* Importer - %s" % importer.DESC
+        if len(msg) < 72:
+            msg += (71 - len(msg))*" " + "*\n"
+        sys.stdout.write(msg)
+        sys.stdout.write("*" * 72 + "\n\n")
+        sys.stdout.write("Processing...")
+        with open(filename) as csv_file:
+            encodings = [settings.ENCODING, settings.ALT_ENCODING, 'utf-8']
+            for encoding in encodings:
+                try:
+                    importer.importation([line for line in
+                            unicodecsv.reader(csv_file, encoding='utf-8')])
+                    errors = importer.get_csv_errors()
+                    sys.stdout.write("\n")
+                    if errors:
+                        print errors
+                        now = datetime.datetime.now().isoformat('-'
+                                                ).replace(':','')
+                        error_file = '.'.join(filename.split('.')[:-1]) \
+                                     + "_errors_%s.csv" % now
+                        sys.stdout.write("Some errors as occured during the ")
+                        sys.stdout.write("import.\n")
+                        try:
+                            with open(error_file, 'w') as fle:
+                                fle.write(errors.encode('utf-8'))
+                            sys.stdout.write("A report has been create in file:"\
+                                             " \"%s\"" % error_file)
+                        except IOError:
+                            sys.stdout.write("Cannot create CSV error file \"%s\"." %
+                                             error_file)
+                    break
+                except ImporterError, e:
+                    if e.type == ImporterError.HEADER and encoding != encodings[-1]:
+                        csv_file.seek(0)
+                        continue
+                except UnicodeDecodeError:
+                    if encoding != encodings[-1]:
+                        csv_file.seek(0)
+                        continue
+        sys.stdout.write("\n\n")
+