diff options
Diffstat (limited to 'archaeological_operations/management')
| -rw-r--r-- | archaeological_operations/management/commands/ishtar_imports.py | 128 | ||||
| -rw-r--r-- | archaeological_operations/management/commands/reindex_acts.py | 59 |
2 files changed, 0 insertions, 187 deletions
diff --git a/archaeological_operations/management/commands/ishtar_imports.py b/archaeological_operations/management/commands/ishtar_imports.py deleted file mode 100644 index 3f4d9e2e8..000000000 --- a/archaeological_operations/management/commands/ishtar_imports.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (C) 2015 Étienne Loks <etienne.loks_AT_peacefrogsDOTnet> - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -# See the file COPYING for details. - -import datetime, unicodecsv -from optparse import make_option - -from django.conf import settings -from django.core.management.base import BaseCommand, CommandError - -from archaeological_operations.data_importer import * - -IMPORTERS = { - 'bibracte-operation':OperationImporterBibracte, - 'bibracte-parcelle':ParcelImporterBibracte, - 'bibracte-docs':DocImporterBibracte, - } - -try: - from archaeological_files.data_importer import * - IMPORTERS['sra-pdl-files'] = FileImporterSraPdL -except ImportError: - pass - -try: - from archaeological_context_records.data_importer import * - IMPORTERS['bibracte-ue'] = ContextRecordsImporterBibracte - IMPORTERS['bibracte-ue-rel'] = ContextRecordsRelationImporterBibracte -except ImportError: - pass - -try: - from archaeological_finds.data_importer import * - IMPORTERS['bibracte-finds'] = FindsImporterBibracte - IMPORTERS['bibracte-finds-alt'] = FindAltImporterBibracte - IMPORTERS['bibracte-treatments'] = TreatmentImporterBibracte -except ImportError: - pass - -class Command(BaseCommand): - args = '<filename> <importer_name> [<nb lines skipped>]' - help = "Import archaeological operations" - option_list = BaseCommand.option_list + ( - make_option('--choose-default', - action='store_true', - dest='choose_default', - default=False, - help='When a choice is requested choose the first one available. '\ - 'For testing purpose'), - ) - - def handle(self, *args, **options): - if not args or not args[0]: - raise CommandError("No file provided.") - if len(args) < 2 or args[1] not in IMPORTERS: - msg = "Bad importer. \nAvailable importers are:\n" - for key in sorted(IMPORTERS.keys()): - msg += "\t* %s: %s\n" % (key, IMPORTERS[key].DESC.encode('utf-8') - or "-") - raise CommandError(msg) - try: - skip_lines = int(args[2]) - except: - skip_lines = 0 - choose_default = options.get('choose_default') - filename = args[0] - importer = IMPORTERS[args[1]](skip_lines=skip_lines, output='cli') - sys.stdout.write("*" * 72 + "\n") - msg = "* Importer - %s" % importer.DESC - if len(msg) < 72: - msg += (71 - len(msg))*" " + "*\n" - sys.stdout.write(msg) - sys.stdout.write("*" * 72 + "\n\n") - sys.stdout.write("Processing...") - with open(filename) as csv_file: - encodings = [settings.ENCODING, settings.ALT_ENCODING, 'utf-8'] - for encoding in encodings: - try: - importer.importation([line for line in - unicodecsv.reader(csv_file, encoding='utf-8')], - choose_default=choose_default) - errors = importer.get_csv_errors() - sys.stdout.write("\n") - if errors: - print errors - now = datetime.datetime.now().isoformat('-' - ).replace(':','') - error_file = '.'.join(filename.split('.')[:-1]) \ - + "_errors_%s.csv" % now - sys.stdout.write("Some errors as occured during the ") - sys.stdout.write("import.\n") - try: - with open(error_file, 'w') as fle: - fle.write(errors.encode('utf-8')) - sys.stdout.write("A report has been create in file:"\ - " \"%s\"" % error_file) - except IOError: - sys.stdout.write("Cannot create CSV error file \"%s\"." % - error_file) - sys.stdout.write( - "\n\n* %d item(s) updated, %d item(s) created.\n" % ( - importer.number_updated, importer.number_created)) - break - except ImporterError, e: - if e.type == ImporterError.HEADER and encoding != encodings[-1]: - csv_file.seek(0) - continue - except UnicodeDecodeError: - if encoding != encodings[-1]: - csv_file.seek(0) - continue - sys.stdout.write("\n\n") - diff --git a/archaeological_operations/management/commands/reindex_acts.py b/archaeological_operations/management/commands/reindex_acts.py deleted file mode 100644 index a701c3fb7..000000000 --- a/archaeological_operations/management/commands/reindex_acts.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (C) 2014 Étienne Loks <etienne.loks_AT_peacefrogsDOTnet> - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -# See the file COPYING for details. - -import sys - -from django.db import models -from django.core.management.base import BaseCommand, CommandError -from django.core.exceptions import ValidationError - -from archaeological_operations import models - -class Command(BaseCommand): - args = '' - help = 'Regenerate index' - - def handle(self, *args, **options): - # first handle not appropriate index - q = models.AdministrativeAct.objects.filter( - signature_date__isnull=False, - index__isnull=False - ).exclude(act_type__indexed=True) - ln = q.count() - sys.stdout.write("\n* clean\n") - for idx, admin_act in enumerate(q.all()): - admin_act.index = None - admin_act.save() - sys.stdout.write("\r* %d/%d" % (idx, ln)) - # then try to fix - q = models.AdministrativeAct.objects - ln = q.count() - sys.stdout.write("\n* fix\n") - tobefix = [] - for idx, admin_act in enumerate(q.all()): - sys.stdout.write("\r* %d/%d" % (idx, ln)) - try: - admin_act.save() - except ValidationError: - tobefix.append(admin_act) - print "To be fix" - for item in tobefix: - print u";".join([unicode(v) for v in (item.signature_date.year, - item.index, item.act_type, - item.operation or item.associated_file)]) |