#!/usr/bin/env python # -*- coding: utf-8 -*- # Copyright (C) 2015 Étienne Loks # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as # published by the Free Software Foundation, either version 3 of the # License, or (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . # See the file COPYING for details. import datetime, unicodecsv from django.conf import settings from django.core.management.base import BaseCommand, CommandError from archaeological_operations.data_importer import * IMPORTERS = { 'bibracte-operation':OperationImporterBibracte, 'bibracte-parcelle':ParcelImporterBibracte, 'bibracte-docs':DocImporterBibracte, } try: from archaeological_context_records.data_importer import * IMPORTERS['bibracte-ue'] = ContextRecordsImporterBibracte IMPORTERS['bibracte-ue-rel'] = ContextRecordsRelationImporterBibracte except ImportError: pass try: from archaeological_finds.data_importer import * IMPORTERS['bibracte-finds'] = FindsImporterBibracte IMPORTERS['bibracte-finds-alt'] = FindAltImporterBibracte IMPORTERS['bibracte-treatments'] = TreatmentImporterBibracte except ImportError: pass class Command(BaseCommand): args = ' []' help = "Import archaeological operations" def handle(self, *args, **options): if not args or not args[0]: raise CommandError("No file provided.") if len(args) < 2 or args[1] not in IMPORTERS: msg = "Bad importer. \nAvailable importers are:\n" for key in sorted(IMPORTERS.keys()): msg += "\t* %s: %s\n" % (key, IMPORTERS[key].DESC.encode('utf-8') or "-") raise CommandError(msg) try: skip_lines = int(args[2]) except: skip_lines = 0 filename = args[0] importer = IMPORTERS[args[1]](skip_lines=skip_lines, output='cli') sys.stdout.write("*" * 72 + "\n") msg = "* Importer - %s" % importer.DESC if len(msg) < 72: msg += (71 - len(msg))*" " + "*\n" sys.stdout.write(msg) sys.stdout.write("*" * 72 + "\n\n") sys.stdout.write("Processing...") with open(filename) as csv_file: encodings = [settings.ENCODING, settings.ALT_ENCODING, 'utf-8'] for encoding in encodings: try: importer.importation([line for line in unicodecsv.reader(csv_file, encoding='utf-8')]) errors = importer.get_csv_errors() sys.stdout.write("\n") if errors: print errors now = datetime.datetime.now().isoformat('-' ).replace(':','') error_file = '.'.join(filename.split('.')[:-1]) \ + "_errors_%s.csv" % now sys.stdout.write("Some errors as occured during the ") sys.stdout.write("import.\n") try: with open(error_file, 'w') as fle: fle.write(errors.encode('utf-8')) sys.stdout.write("A report has been create in file:"\ " \"%s\"" % error_file) except IOError: sys.stdout.write("Cannot create CSV error file \"%s\"." % error_file) sys.stdout.write( "\n\n* %d item(s) updated, %d item(s) created.\n" % ( importer.number_updated, importer.number_created)) break except ImporterError, e: if e.type == ImporterError.HEADER and encoding != encodings[-1]: csv_file.seek(0) continue except UnicodeDecodeError: if encoding != encodings[-1]: csv_file.seek(0) continue sys.stdout.write("\n\n")