diff options
Diffstat (limited to 'archaeological_operations/management/commands/import_operations.py')
| -rw-r--r--[-rwxr-xr-x] | archaeological_operations/management/commands/import_operations.py | 91 |
1 files changed, 62 insertions, 29 deletions
diff --git a/archaeological_operations/management/commands/import_operations.py b/archaeological_operations/management/commands/import_operations.py index a9ecf41c9..fe4afc032 100755..100644 --- a/archaeological_operations/management/commands/import_operations.py +++ b/archaeological_operations/management/commands/import_operations.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (C) 2012-2013 Étienne Loks <etienne.loks_AT_peacefrogsDOTnet> +# Copyright (C) 2015 Étienne Loks <etienne.loks_AT_peacefrogsDOTnet> # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -17,40 +17,73 @@ # See the file COPYING for details. +import datetime, unicodecsv + +from django.conf import settings from django.core.management.base import BaseCommand, CommandError -from archaeological_operations.import_from_csv import import_from_csv -from archaeological_operations.import_from_dbf import import_from_dbf -IMPORTERS = {'csv':import_from_csv, - 'dbf':import_from_dbf, - 'db3':import_from_dbf, - 'fp':import_from_dbf, - 'vfp':import_from_dbf} +from archaeological_operations.data_importer import * + +IMPORTERS = {'bibracte-operation':OperationImporterBibracte} class Command(BaseCommand): - args = '<filename> [<lines>]' - help = "Import archaelogical operations" + args = '<filename> <importer_name> [<nb lines skipped>]' + help = "Import archaeological operations" def handle(self, *args, **options): if not args or not args[0]: raise CommandError("No file provided.") + if len(args) < 2 or args[1] not in IMPORTERS: + msg = "Bad importer. \nAvailable importers are:\n" + for key in IMPORTERS: + msg += "\t* %s: %s" % (key, IMPORTERS[key].DESC.encode('utf-8') + or "-") + raise CommandError(msg) + try: + skip_lines = int(args[2]) + except: + skip_lines = 0 filename = args[0] - update = True - file_type = None - lines = len(args) > 1 and args[1] - if not file_type: - suffix = filename.split('.')[-1].lower() - if suffix in IMPORTERS.keys(): - file_type = suffix - else: - raise CommandError("This file extension is not managed. "\ - "Specify manualy the file type.") - elif file_type not in IMPORTERS.keys(): - raise CommandError("This file type is not managed.") - nb_ops, errors = IMPORTERS[file_type](filename, - update=update, - stdout=self.stdout, - lines=lines) - self.stdout.write('\n* %d operation treated\n' % nb_ops) - if errors: - self.stderr.write('\n'.join(errors)) + importer = IMPORTERS[args[1]](skip_lines=skip_lines, output='cli') + sys.stdout.write("*" * 72 + "\n") + msg = "* Importer - %s" % importer.DESC + if len(msg) < 72: + msg += (71 - len(msg))*" " + "*\n" + sys.stdout.write(msg) + sys.stdout.write("*" * 72 + "\n\n") + sys.stdout.write("Processing...") + with open(filename) as csv_file: + encodings = [settings.ENCODING, settings.ALT_ENCODING, 'utf-8'] + for encoding in encodings: + try: + importer.importation([line for line in + unicodecsv.reader(csv_file, encoding='utf-8')]) + errors = importer.get_csv_errors() + sys.stdout.write("\n") + if errors: + print errors + now = datetime.datetime.now().isoformat('-' + ).replace(':','') + error_file = '.'.join(filename.split('.')[:-1]) \ + + "_errors_%s.csv" % now + sys.stdout.write("Some errors as occured during the ") + sys.stdout.write("import.\n") + try: + with open(error_file, 'w') as fle: + fle.write(errors.encode('utf-8')) + sys.stdout.write("A report has been create in file:"\ + " \"%s\"" % error_file) + except IOError: + sys.stdout.write("Cannot create CSV error file \"%s\"." % + error_file) + break + except ImporterError, e: + if e.type == ImporterError.HEADER and encoding != encodings[-1]: + csv_file.seek(0) + continue + except UnicodeDecodeError: + if encoding != encodings[-1]: + csv_file.seek(0) + continue + sys.stdout.write("\n\n") + |