From cadd34a00816a28de6002de7396256b9eaa14531 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Wed, 18 Feb 2015 22:38:05 +0100 Subject: Improve archaeological files import --- .../management/commands/import_operations.py | 111 ------------------- .../management/commands/ishtar_imports.py | 117 +++++++++++++++++++++ 2 files changed, 117 insertions(+), 111 deletions(-) delete mode 100644 archaeological_operations/management/commands/import_operations.py create mode 100644 archaeological_operations/management/commands/ishtar_imports.py (limited to 'archaeological_operations/management') diff --git a/archaeological_operations/management/commands/import_operations.py b/archaeological_operations/management/commands/import_operations.py deleted file mode 100644 index 09bfe23b6..000000000 --- a/archaeological_operations/management/commands/import_operations.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (C) 2015 Étienne Loks - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -# See the file COPYING for details. - -import datetime, unicodecsv - -from django.conf import settings -from django.core.management.base import BaseCommand, CommandError - -from archaeological_operations.data_importer import * - -IMPORTERS = { - 'bibracte-operation':OperationImporterBibracte, - 'bibracte-parcelle':ParcelImporterBibracte, - 'bibracte-docs':DocImporterBibracte, - } - -try: - from archaeological_context_records.data_importer import * - IMPORTERS['bibracte-ue'] = ContextRecordsImporterBibracte - IMPORTERS['bibracte-ue-rel'] = ContextRecordsRelationImporterBibracte -except ImportError: - pass - -try: - from archaeological_finds.data_importer import * - IMPORTERS['bibracte-finds'] = FindsImporterBibracte - IMPORTERS['bibracte-finds-alt'] = FindAltImporterBibracte - IMPORTERS['bibracte-treatments'] = TreatmentImporterBibracte -except ImportError: - pass - -class Command(BaseCommand): - args = ' []' - help = "Import archaeological operations" - - def handle(self, *args, **options): - if not args or not args[0]: - raise CommandError("No file provided.") - if len(args) < 2 or args[1] not in IMPORTERS: - msg = "Bad importer. \nAvailable importers are:\n" - for key in sorted(IMPORTERS.keys()): - msg += "\t* %s: %s\n" % (key, IMPORTERS[key].DESC.encode('utf-8') - or "-") - raise CommandError(msg) - try: - skip_lines = int(args[2]) - except: - skip_lines = 0 - filename = args[0] - importer = IMPORTERS[args[1]](skip_lines=skip_lines, output='cli') - sys.stdout.write("*" * 72 + "\n") - msg = "* Importer - %s" % importer.DESC - if len(msg) < 72: - msg += (71 - len(msg))*" " + "*\n" - sys.stdout.write(msg) - sys.stdout.write("*" * 72 + "\n\n") - sys.stdout.write("Processing...") - with open(filename) as csv_file: - encodings = [settings.ENCODING, settings.ALT_ENCODING, 'utf-8'] - for encoding in encodings: - try: - importer.importation([line for line in - unicodecsv.reader(csv_file, encoding='utf-8')]) - errors = importer.get_csv_errors() - sys.stdout.write("\n") - if errors: - print errors - now = datetime.datetime.now().isoformat('-' - ).replace(':','') - error_file = '.'.join(filename.split('.')[:-1]) \ - + "_errors_%s.csv" % now - sys.stdout.write("Some errors as occured during the ") - sys.stdout.write("import.\n") - try: - with open(error_file, 'w') as fle: - fle.write(errors.encode('utf-8')) - sys.stdout.write("A report has been create in file:"\ - " \"%s\"" % error_file) - except IOError: - sys.stdout.write("Cannot create CSV error file \"%s\"." % - error_file) - sys.stdout.write( - "\n\n* %d item(s) updated, %d item(s) created.\n" % ( - importer.number_updated, importer.number_created)) - break - except ImporterError, e: - if e.type == ImporterError.HEADER and encoding != encodings[-1]: - csv_file.seek(0) - continue - except UnicodeDecodeError: - if encoding != encodings[-1]: - csv_file.seek(0) - continue - sys.stdout.write("\n\n") - diff --git a/archaeological_operations/management/commands/ishtar_imports.py b/archaeological_operations/management/commands/ishtar_imports.py new file mode 100644 index 000000000..23397204b --- /dev/null +++ b/archaeological_operations/management/commands/ishtar_imports.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (C) 2015 Étienne Loks + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# See the file COPYING for details. + +import datetime, unicodecsv + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError + +from archaeological_operations.data_importer import * + +IMPORTERS = { + 'bibracte-operation':OperationImporterBibracte, + 'bibracte-parcelle':ParcelImporterBibracte, + 'bibracte-docs':DocImporterBibracte, + } + +try: + from archaeological_files.data_importer import * + IMPORTERS['sra-pdl-files'] = FileImporterSraPdL +except ImportError: + pass + +try: + from archaeological_context_records.data_importer import * + IMPORTERS['bibracte-ue'] = ContextRecordsImporterBibracte + IMPORTERS['bibracte-ue-rel'] = ContextRecordsRelationImporterBibracte +except ImportError: + pass + +try: + from archaeological_finds.data_importer import * + IMPORTERS['bibracte-finds'] = FindsImporterBibracte + IMPORTERS['bibracte-finds-alt'] = FindAltImporterBibracte + IMPORTERS['bibracte-treatments'] = TreatmentImporterBibracte +except ImportError: + pass + +class Command(BaseCommand): + args = ' []' + help = "Import archaeological operations" + + def handle(self, *args, **options): + if not args or not args[0]: + raise CommandError("No file provided.") + if len(args) < 2 or args[1] not in IMPORTERS: + msg = "Bad importer. \nAvailable importers are:\n" + for key in sorted(IMPORTERS.keys()): + msg += "\t* %s: %s\n" % (key, IMPORTERS[key].DESC.encode('utf-8') + or "-") + raise CommandError(msg) + try: + skip_lines = int(args[2]) + except: + skip_lines = 0 + filename = args[0] + importer = IMPORTERS[args[1]](skip_lines=skip_lines, output='cli') + sys.stdout.write("*" * 72 + "\n") + msg = "* Importer - %s" % importer.DESC + if len(msg) < 72: + msg += (71 - len(msg))*" " + "*\n" + sys.stdout.write(msg) + sys.stdout.write("*" * 72 + "\n\n") + sys.stdout.write("Processing...") + with open(filename) as csv_file: + encodings = [settings.ENCODING, settings.ALT_ENCODING, 'utf-8'] + for encoding in encodings: + try: + importer.importation([line for line in + unicodecsv.reader(csv_file, encoding='utf-8')]) + errors = importer.get_csv_errors() + sys.stdout.write("\n") + if errors: + print errors + now = datetime.datetime.now().isoformat('-' + ).replace(':','') + error_file = '.'.join(filename.split('.')[:-1]) \ + + "_errors_%s.csv" % now + sys.stdout.write("Some errors as occured during the ") + sys.stdout.write("import.\n") + try: + with open(error_file, 'w') as fle: + fle.write(errors.encode('utf-8')) + sys.stdout.write("A report has been create in file:"\ + " \"%s\"" % error_file) + except IOError: + sys.stdout.write("Cannot create CSV error file \"%s\"." % + error_file) + sys.stdout.write( + "\n\n* %d item(s) updated, %d item(s) created.\n" % ( + importer.number_updated, importer.number_created)) + break + except ImporterError, e: + if e.type == ImporterError.HEADER and encoding != encodings[-1]: + csv_file.seek(0) + continue + except UnicodeDecodeError: + if encoding != encodings[-1]: + csv_file.seek(0) + continue + sys.stdout.write("\n\n") + -- cgit v1.2.3