1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2015 Étienne Loks <etienne.loks_AT_peacefrogsDOTnet>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# See the file COPYING for details.
import datetime, unicodecsv
from optparse import make_option
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from archaeological_operations.data_importer import *
IMPORTERS = {
'bibracte-operation':OperationImporterBibracte,
'bibracte-parcelle':ParcelImporterBibracte,
'bibracte-docs':DocImporterBibracte,
}
try:
from archaeological_files.data_importer import *
IMPORTERS['sra-pdl-files'] = FileImporterSraPdL
except ImportError:
pass
try:
from archaeological_context_records.data_importer import *
IMPORTERS['bibracte-ue'] = ContextRecordsImporterBibracte
IMPORTERS['bibracte-ue-rel'] = ContextRecordsRelationImporterBibracte
except ImportError:
pass
try:
from archaeological_finds.data_importer import *
IMPORTERS['bibracte-finds'] = FindsImporterBibracte
IMPORTERS['bibracte-finds-alt'] = FindAltImporterBibracte
IMPORTERS['bibracte-treatments'] = TreatmentImporterBibracte
except ImportError:
pass
class Command(BaseCommand):
args = '<filename> <importer_name> [<nb lines skipped>]'
help = "Import archaeological operations"
option_list = BaseCommand.option_list + (
make_option('--choose-default',
action='store_true',
dest='choose_default',
default=False,
help='When a choice is requested choose the first one available. '\
'For testing purpose'),
)
def handle(self, *args, **options):
if not args or not args[0]:
raise CommandError("No file provided.")
if len(args) < 2 or args[1] not in IMPORTERS:
msg = "Bad importer. \nAvailable importers are:\n"
for key in sorted(IMPORTERS.keys()):
msg += "\t* %s: %s\n" % (key, IMPORTERS[key].DESC.encode('utf-8')
or "-")
raise CommandError(msg)
try:
skip_lines = int(args[2])
except:
skip_lines = 0
choose_default = options.get('choose_default')
filename = args[0]
importer = IMPORTERS[args[1]](skip_lines=skip_lines, output='cli')
sys.stdout.write("*" * 72 + "\n")
msg = "* Importer - %s" % importer.DESC
if len(msg) < 72:
msg += (71 - len(msg))*" " + "*\n"
sys.stdout.write(msg)
sys.stdout.write("*" * 72 + "\n\n")
sys.stdout.write("Processing...")
with open(filename) as csv_file:
encodings = [settings.ENCODING, settings.ALT_ENCODING, 'utf-8']
for encoding in encodings:
try:
importer.importation([line for line in
unicodecsv.reader(csv_file, encoding='utf-8')],
choose_default=choose_default)
errors = importer.get_csv_errors()
sys.stdout.write("\n")
if errors:
print errors
now = datetime.datetime.now().isoformat('-'
).replace(':','')
error_file = '.'.join(filename.split('.')[:-1]) \
+ "_errors_%s.csv" % now
sys.stdout.write("Some errors as occured during the ")
sys.stdout.write("import.\n")
try:
with open(error_file, 'w') as fle:
fle.write(errors.encode('utf-8'))
sys.stdout.write("A report has been create in file:"\
" \"%s\"" % error_file)
except IOError:
sys.stdout.write("Cannot create CSV error file \"%s\"." %
error_file)
sys.stdout.write(
"\n\n* %d item(s) updated, %d item(s) created.\n" % (
importer.number_updated, importer.number_created))
break
except ImporterError, e:
if e.type == ImporterError.HEADER and encoding != encodings[-1]:
csv_file.seek(0)
continue
except UnicodeDecodeError:
if encoding != encodings[-1]:
csv_file.seek(0)
continue
sys.stdout.write("\n\n")
|
