diff options
| -rw-r--r-- | archaeological_files/data_importer.py | 51 | ||||
| -rw-r--r-- | archaeological_files/models.py | 9 | ||||
| -rw-r--r-- | archaeological_files_pdl/forms.py | 4 | ||||
| -rw-r--r-- | archaeological_files_pdl/views.py | 6 | ||||
| -rw-r--r-- | example_project/settings.py | 2 | ||||
| -rw-r--r-- | ishtar_common/data_importer.py | 128 | ||||
| -rw-r--r-- | ishtar_common/forms_common.py | 2 | ||||
| -rw-r--r-- | ishtar_common/migrations/0041_auto__add_field_importertype_unicity_keys__add_field_importtarget_conc.py | 353 | ||||
| -rw-r--r-- | ishtar_common/models.py | 16 | ||||
| -rw-r--r-- | ishtar_common/wizards.py | 2 | ||||
| -rw-r--r-- | misc/pre_import_sra_files.py | 79 |
11 files changed, 567 insertions, 85 deletions
diff --git a/archaeological_files/data_importer.py b/archaeological_files/data_importer.py index f3268d24d..a5fa3c18d 100644 --- a/archaeological_files/data_importer.py +++ b/archaeological_files/data_importer.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (C) 2013-2014 Étienne Loks <etienne.loks_AT_peacefrogsDOTnet> +# Copyright (C) 2013-2015 Étienne Loks <etienne.loks_AT_peacefrogsDOTnet> # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -58,40 +58,7 @@ class ImportMayorFormater(ImportFormater): class FilePostProcessing(object): - # erroneaous date to be easily identified on post-processing - DEFAULT_YEAR = 1871 - EXTRA_DEFAULTS = {tuple(): {'year': DEFAULT_YEAR}} - def post_processing(self, item, data): - if item.year == self.DEFAULT_YEAR: # the default year has been applied - if item.creation_date: - item.year = item.creation_date.year - elif item.reception_date: - item.year = item.reception_date.year - if item.external_id: - idx = None - if '-' in item.external_id: - year, y_idx = item.external_id.split('-') - if len(year) == 4: # 2007-XXXX - try: - item.year = int(year) - idx = y_idx - except ValueError: - pass - if '.' in item.external_id: - year, y_idx = item.external_id.split('.') - if len(year) == 4: # 2011.XXXX - try: - item.year = int(year) - idx = y_idx - except ValueError: - pass - if not idx: - idx = item.external_id - try: - item.numeric_reference = int(idx) - except ValueError: - pass if not item.end_date: # auto-close open_date = item.reception_date or item.creation_date item.end_date = open_date + datetime.timedelta(30) @@ -104,6 +71,7 @@ class FileImporterSraPdL(FilePostProcessing, Importer): SLUG = "sra-pdl-files" LINE_FORMAT = [] OBJECT_CLS = models.File + UNICITY_KEYS = ['external_id'] DEFAULTS = { ('responsible_town_planning_service', 'attached_to'): { 'organization_type': OrganizationType.objects.get( @@ -228,8 +196,7 @@ class FileImporterSraPdL(FilePostProcessing, Importer): ImportFormater('internal_reference', # AL, 38 UnicodeFormater(60), comment=u"Autre référence", - required=False, - duplicate_fields=[['external_id', False]]), + required=False), None, # AM, 39 None, # AN, 40 ImportFormater('comment', # AO, 41 @@ -337,6 +304,18 @@ class FileImporterSraPdL(FilePostProcessing, Importer): required=False, comment=u"Type de permis"), # CG, 85 None, # CH, 85 + ImportFormater('year', # CI, 86 + IntegerFormater(), + comment=u"Année du dossier", + required=True), + ImportFormater('numeric_reference', # CJ, 87 + IntegerFormater(), + comment=u"Identifiant numérique", + required=True), + ImportFormater('external_id', # CK, 88 + UnicodeFormater(), + comment=u"Identifiant externe", + required=True), ] def __init__(self, *args, **kwargs): diff --git a/archaeological_files/models.py b/archaeological_files/models.py index 72da0eb5f..a442d99c2 100644 --- a/archaeological_files/models.py +++ b/archaeological_files/models.py @@ -410,6 +410,12 @@ class File(BaseHistorizedItem, OwnPerms, ValueGetter, ShortMenuItem, self.update_resp_planning_service() updated += self.update_raw_general_contractor() updated += self.update_corpo_general_contractor() + if self.year and self.numeric_reference: + external_id = u"{}{}-{}".format(settings.ISHTAR_LOCAL_PREFIX, + self.year, self.numeric_reference) + if external_id != self.external_id: + updated = True + self.external_id = external_id if updated: self.save() self.update_delay_date() @@ -435,6 +441,9 @@ class FileByDepartment(models.Model): managed = False db_table = 'file_department' + def __unicode__(self): + return u"{} - {}".format(self.file, self.department) + class FileDashboard: def __init__(self): diff --git a/archaeological_files_pdl/forms.py b/archaeological_files_pdl/forms.py index 0adfa087a..84931996a 100644 --- a/archaeological_files_pdl/forms.py +++ b/archaeological_files_pdl/forms.py @@ -309,8 +309,12 @@ class FileFormInstruction(forms.Form): c_year = datetime.date.today().year if 'year' in kwargs: c_year = kwargs.pop('year') + if 'data' in kwargs: + kwargs['data'][kwargs.get('prefix', '') + '-year'] = c_year + super(FileFormInstruction, self).__init__(*args, **kwargs) self.fields['year'].initial = c_year + self.fields['year'].widget.attrs.update({'readonly': 'readonly'}) c_num, lasts = 0, "" q = models.File.objects.filter(numeric_reference__isnull=False, diff --git a/archaeological_files_pdl/views.py b/archaeological_files_pdl/views.py index 8305bdab7..b3acc12b1 100644 --- a/archaeological_files_pdl/views.py +++ b/archaeological_files_pdl/views.py @@ -55,8 +55,7 @@ file_creation_wizard = FileWizard.as_view([ 'planningservice-file_creation': file_creation_wizard_is_preventive, 'researchaddress-file_creation': file_creation_wizard_is_not_preventive, - 'research-file_creation': file_creation_wizard_is_not_preventive - }, + 'research-file_creation': file_creation_wizard_is_not_preventive}, url_name='file_creation',) file_modification_wizard_is_preventive = is_preventive( @@ -86,8 +85,7 @@ file_modification_wizard = FileModificationWizard.as_view([ file_modification_wizard_is_preventive, 'researchaddress-file_modification': file_modification_wizard_is_not_preventive, - 'research-file_modification': file_modification_wizard_is_not_preventive - }, + 'research-file_modification': file_modification_wizard_is_not_preventive}, url_name='file_modification',) diff --git a/example_project/settings.py b/example_project/settings.py index c669ea954..f4d7ca609 100644 --- a/example_project/settings.py +++ b/example_project/settings.py @@ -190,6 +190,8 @@ ISHTAR_PERIODS = {} ISHTAR_PERMIT_TYPES = {} ISHTAR_DOC_TYPES = {u"undefined": u"Undefined"} +ISHTAR_LOCAL_PREFIX = "SRA" + ISHTAR_DPTS = [] OP_PREFIX = 'OP' diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index 6239bf4c0..3d623e7a6 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -177,7 +177,8 @@ class UnicodeFormater(Formater): 'length': self.max_length}) if self.notnull and not value: return - value = self.prefix + value + if value: + value = self.prefix + value return value @@ -603,7 +604,6 @@ class Importer(object): UNICITY_KEYS = [] EXTRA_DEFAULTS = {} DEFAULTS = {} - STR_CUT = {} ERRORS = { 'header_check': _( u"The given file is not correct. Check the file " @@ -630,9 +630,13 @@ class Importer(object): model_name = self.OBJECT_CLS.__module__ + '.' + \ self.OBJECT_CLS.__name__ + unicity_keys = '' + if self.UNICITY_KEYS: + unicity_keys = ";".join(self.UNICITY_KEYS) + importer = models.ImporterType.objects.create( slug=self.SLUG, name=name, description=self.DESC, - associated_models=model_name) + associated_models=model_name, unicity_keys=unicity_keys) for default in self.DEFAULTS: values = self.DEFAULTS[default] @@ -691,13 +695,15 @@ class Importer(object): models.ImportTarget.objects.get_or_create( column=column, target=target, formater_type=formater_model, force_new=getattr(formater, 'force_new', False), + concat=getattr(formater, 'concat', False), regexp_filter=regexp_filter, comment=line.comment) return True def __init__(self, skip_lines=0, reference_header=None, check_col_num=False, test=False, history_modifier=None, - output='silent', import_instance=None): + output='silent', import_instance=None, + conservative_import=False): """ * skip_line must be set if the data provided has got headers lines. * a reference_header can be provided to perform a data compliance @@ -715,8 +721,12 @@ class Importer(object): self.line_format = copy.copy(self.LINE_FORMAT) self.import_instance = import_instance self.archive = None + self.conservative_import = conservative_import + # for a conservative_import UNICITY_KEYS should be defined + assert not self.conservative_import or bool(self.UNICITY_KEYS) self.DB_TARGETS = {} self.match_table = {} + self.concats = set() if import_instance and import_instance.imported_images: self.archive = import_instance.imported_images self._defaults = self.DEFAULTS.copy() @@ -883,6 +893,7 @@ class Importer(object): self.errors.append((idx_line, None, msg)) def _line_processing(self, idx_line, line): + self.idx_line = idx_line if self.skip_lines > idx_line: self.validity.append(line) return @@ -927,17 +938,11 @@ class Importer(object): if self.test: return # manage unicity of items (mainly for updates) - if self.UNICITY_KEYS: - data['defaults'] = {} - for k in data.keys(): - if k not in self.UNICITY_KEYS \ - and k != 'defaults': - data['defaults'][k] = data.pop(k) - if 'history_modifier' in \ self.OBJECT_CLS._meta.get_all_field_names(): data['history_modifier'] = self.history_modifier + self.plouf = 0 obj, created = self.get_object(self.OBJECT_CLS, data) if self.import_instance and hasattr(obj, 'imports') \ and created: @@ -1014,10 +1019,10 @@ class Importer(object): self.errors.append( (idx_line + 1, idx_col + 1, self.ERRORS['value_required'])) + self.c_errors = True elif not val.strip(): c_row.append("") return - self.c_errors = True val = val.replace(NEW_LINE_BREAK, '\n') self.errors.append( (idx_line + 1, idx_col + 1, @@ -1049,6 +1054,8 @@ class Importer(object): field_name = field_name[idx_v] if type(force_new) in (list, tuple): force_new = force_new[idx_v] + if formater.concat: + self.concats.add(field_name) if self.DB_TARGETS: formater.reinit_db_target( @@ -1192,14 +1199,18 @@ class Importer(object): m2ms.append((attribute, v)) elif hasattr(field_object, 'rel') and field_object.rel: if type(data[attribute]) == dict: - c_path.append(attribute) # put history_modifier for every created item if 'history_modifier' in \ field_object.rel.to._meta.get_all_field_names(): data[attribute]['history_modifier'] = \ self.history_modifier - data[attribute], created = self.get_object( - field_object.rel.to, data[attribute], c_path) + try: + c_path.append(attribute) + data[attribute], created = self.get_object( + field_object.rel.to, data[attribute].copy(), c_path) + except ImporterError, msg: + self.errors.append((self.idx_line, None, msg)) + data[attribute] = None elif type(data[attribute]) == list: data[attribute] = data[attribute][0] @@ -1207,37 +1218,37 @@ class Importer(object): m2ms = [] if data and type(data) == dict: c_path = path[:] - for attribute in data.keys(): + + # get all related fields + for attribute in list(data.keys()): + c_c_path = c_path[:] if not attribute: data.pop(attribute) continue if not data[attribute]: continue if attribute != '__force_new': - self.get_field(cls, attribute, data, m2ms, c_path) - # default values - path = tuple(path) - if path in self._defaults: - for k in self._defaults[path]: - if k not in data or not data[k]: - data[k] = self._defaults[path][k] - - # pre treatment - if path in self.STR_CUT: - for k in self.STR_CUT[path]: - if k in data and data[k]: - data[k] = unicode(data[k])[:self.STR_CUT[path][k]] + self.get_field(cls, attribute, data, m2ms, c_c_path) - # filter default values + # filter uncessary default values create_dict = copy.deepcopy(data) for k in create_dict.keys(): if type(create_dict[k]) == dict: create_dict.pop(k) + + # default values + path = tuple(path) defaults = {} + if path in self._defaults: + for k in self._defaults[path]: + if (k not in data or not data[k]): + defaults[k] = self._defaults[path][k] + if 'history_modifier' in create_dict: - defaults = { + defaults.update({ 'history_modifier': create_dict.pop('history_modifier') - } + }) + try: try: dct = create_dict.copy() @@ -1248,10 +1259,44 @@ class Importer(object): created = dct.pop('__force_new') if not [k for k in dct if dct[k] is not None]: return None, created - obj = cls.objects.create(**dct) + new_dct = defaults.copy() + new_dct.update(dct) + obj = cls.objects.create(**new_dct) else: - dct['defaults'] = defaults + # manage UNICITY_KEYS - only level 1 + if not path and self.UNICITY_KEYS: + for k in dct.keys(): + if k not in self.UNICITY_KEYS \ + and k != 'defaults': + defaults[k] = dct.pop(k) + + dct['defaults'] = defaults.copy() obj, created = cls.objects.get_or_create(**dct) + + if not created and not path and self.UNICITY_KEYS: + changed = False + if self.conservative_import: + for k in dct['defaults']: + new_val = dct['defaults'][k] + if new_val is None or new_val == '': + continue + val = getattr(obj, k) + if val is None or val == '': + changed = True + setattr(obj, k, new_val) + elif k in self.concats \ + and type(val) == unicode \ + and type(new_val) == unicode: + setattr(obj, k, val + u" - " + new_val) + else: + for k in dct['defaults']: + new_val = dct['defaults'][k] + if new_val is None or new_val == '': + continue + changed = True + setattr(obj, k, new_val) + if changed: + obj.save() if self.import_instance and hasattr(obj, 'imports') \ and created: obj.imports.add(self.import_instance) @@ -1286,24 +1331,25 @@ class Importer(object): except UnicodeDecodeError: data = '' raise ImporterError( - "Erreur d'import %s, contexte : %s, erreur : %s" - % (unicode(cls), unicode(data), message)) + "Erreur d'import %s %s, contexte : %s, erreur : %s" + % (unicode(cls), unicode("__".join(path)), + unicode(data), message)) return obj, created return data - def _format_csv_line(self, values): + def _format_csv_line(self, values, empty=u"-"): return u'"' + u'","'.join( - [(v and unicode(v).replace('"', '""')) or u'-' + [(v and unicode(v).replace('"', '""')) or empty for v in values]) + u'"' - def _get_csv(self, rows, header=[]): + def _get_csv(self, rows, header=[], empty=u"-"): if not rows: return "" csv_v = [] if header: - csv_v.append(self._format_csv_line(header)) + csv_v.append(self._format_csv_line(header, empty=empty)) for values in rows: - csv_v.append(self._format_csv_line(values)) + csv_v.append(self._format_csv_line(values, empty=empty)) return u"\n".join(csv_v) def get_csv_errors(self): diff --git a/ishtar_common/forms_common.py b/ishtar_common/forms_common.py index c3b70d42b..0e6a34b74 100644 --- a/ishtar_common/forms_common.py +++ b/ishtar_common/forms_common.py @@ -104,7 +104,7 @@ class NewImportForm(forms.ModelForm): class Meta: model = models.Import fields = ('importer_type', 'imported_file', 'imported_images', - 'encoding', 'skip_lines') + 'conservative_import', 'encoding', 'skip_lines') def save(self, user, commit=True): self.instance.user = user diff --git a/ishtar_common/migrations/0041_auto__add_field_importertype_unicity_keys__add_field_importtarget_conc.py b/ishtar_common/migrations/0041_auto__add_field_importertype_unicity_keys__add_field_importtarget_conc.py new file mode 100644 index 000000000..990f35efc --- /dev/null +++ b/ishtar_common/migrations/0041_auto__add_field_importertype_unicity_keys__add_field_importtarget_conc.py @@ -0,0 +1,353 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'ImporterType.unicity_keys' + db.add_column('ishtar_common_importertype', 'unicity_keys', + self.gf('django.db.models.fields.CharField')(max_length=500, null=True, blank=True), + keep_default=False) + + # Adding field 'ImportTarget.concat' + db.add_column('ishtar_common_importtarget', 'concat', + self.gf('django.db.models.fields.BooleanField')(default=False), + keep_default=False) + + # Adding field 'Import.conservative_import' + db.add_column('ishtar_common_import', 'conservative_import', + self.gf('django.db.models.fields.BooleanField')(default=False), + keep_default=False) + + + def backwards(self, orm): + # Deleting field 'ImporterType.unicity_keys' + db.delete_column('ishtar_common_importertype', 'unicity_keys') + + # Deleting field 'ImportTarget.concat' + db.delete_column('ishtar_common_importtarget', 'concat') + + # Deleting field 'Import.conservative_import' + db.delete_column('ishtar_common_import', 'conservative_import') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'ishtar_common.arrondissement': { + 'Meta': {'object_name': 'Arrondissement'}, + 'department': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Department']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '30'}) + }, + 'ishtar_common.author': { + 'Meta': {'object_name': 'Author'}, + 'author_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.AuthorType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'person': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'author'", 'to': "orm['ishtar_common.Person']"}) + }, + 'ishtar_common.authortype': { + 'Meta': {'object_name': 'AuthorType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}) + }, + 'ishtar_common.canton': { + 'Meta': {'object_name': 'Canton'}, + 'arrondissement': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Arrondissement']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '30'}) + }, + 'ishtar_common.department': { + 'Meta': {'ordering': "['number']", 'object_name': 'Department'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '30'}), + 'number': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '3'}), + 'state': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.State']", 'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.documenttemplate': { + 'Meta': {'ordering': "['associated_object_name', 'name']", 'object_name': 'DocumentTemplate'}, + 'associated_object_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'template': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}) + }, + 'ishtar_common.format': { + 'Meta': {'object_name': 'Format'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}) + }, + 'ishtar_common.formatertype': { + 'Meta': {'ordering': "('formater_type', 'options')", 'unique_together': "(('formater_type', 'options', 'many_split'),)", 'object_name': 'FormaterType'}, + 'formater_type': ('django.db.models.fields.CharField', [], {'max_length': '20'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'many_split': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}), + 'options': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.globalvar': { + 'Meta': {'ordering': "['slug']", 'object_name': 'GlobalVar'}, + 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '50'}), + 'value': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.historicalorganization': { + 'Meta': {'ordering': "('-history_date', '-history_id')", 'object_name': 'HistoricalOrganization'}, + 'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'history_creator_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'history_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'history_id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'history_modifier_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'history_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'history_user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True'}), + 'id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'blank': 'True'}), + 'merge_key': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}), + 'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '300'}), + 'organization_type_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}), + 'town': ('django.db.models.fields.CharField', [], {'max_length': '70', 'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.import': { + 'Meta': {'object_name': 'Import'}, + 'conservative_import': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'creation_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'null': 'True', 'blank': 'True'}), + 'encoding': ('django.db.models.fields.CharField', [], {'default': "'utf-8'", 'max_length': '15'}), + 'end_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'error_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'imported_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}), + 'imported_images': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'importer_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.ImporterType']"}), + 'match_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'result_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'seconds_remaining': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'skip_lines': ('django.db.models.fields.IntegerField', [], {'default': '1'}), + 'state': ('django.db.models.fields.CharField', [], {'default': "'C'", 'max_length': '2'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.IshtarUser']"}) + }, + 'ishtar_common.importercolumn': { + 'Meta': {'object_name': 'ImporterColumn'}, + 'col_number': ('django.db.models.fields.IntegerField', [], {'default': '1'}), + 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'importer_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'columns'", 'to': "orm['ishtar_common.ImporterType']"}), + 'regexp_pre_filter': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Regexp']", 'null': 'True', 'blank': 'True'}), + 'required': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + }, + 'ishtar_common.importerdefault': { + 'Meta': {'object_name': 'ImporterDefault'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'importer_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'defaults'", 'to': "orm['ishtar_common.ImporterType']"}), + 'target': ('django.db.models.fields.CharField', [], {'max_length': '500'}) + }, + 'ishtar_common.importerdefaultvalues': { + 'Meta': {'object_name': 'ImporterDefaultValues'}, + 'default_target': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'default_values'", 'to': "orm['ishtar_common.ImporterDefault']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'target': ('django.db.models.fields.CharField', [], {'max_length': '500'}), + 'value': ('django.db.models.fields.CharField', [], {'max_length': '500'}) + }, + 'ishtar_common.importerduplicatefield': { + 'Meta': {'object_name': 'ImporterDuplicateField'}, + 'column': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'duplicate_fields'", 'to': "orm['ishtar_common.ImporterColumn']"}), + 'field_name': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'force_new': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + 'ishtar_common.importertype': { + 'Meta': {'object_name': 'ImporterType'}, + 'associated_models': ('django.db.models.fields.CharField', [], {'max_length': '200'}), + 'description': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_template': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'slug': ('django.db.models.fields.SlugField', [], {'max_length': '100', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + 'unicity_keys': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}), + 'users': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': "orm['ishtar_common.IshtarUser']", 'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.importtarget': { + 'Meta': {'object_name': 'ImportTarget'}, + 'column': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'targets'", 'to': "orm['ishtar_common.ImporterColumn']"}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'concat': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'force_new': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'formater_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.FormaterType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'regexp_filter': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Regexp']", 'null': 'True', 'blank': 'True'}), + 'target': ('django.db.models.fields.CharField', [], {'max_length': '500'}) + }, + 'ishtar_common.ishtaruser': { + 'Meta': {'object_name': 'IshtarUser', '_ormbases': ['auth.User']}, + 'person': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'ishtaruser'", 'unique': 'True', 'to': "orm['ishtar_common.Person']"}), + 'user_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'ishtar_common.itemkey': { + 'Meta': {'object_name': 'ItemKey'}, + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'importer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Import']", 'null': 'True', 'blank': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}) + }, + 'ishtar_common.organization': { + 'Meta': {'object_name': 'Organization'}, + 'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'history_creator': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['auth.User']"}), + 'history_modifier': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['auth.User']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'imports': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'imported_ishtar_common_organization'", 'null': 'True', 'symmetrical': 'False', 'to': "orm['ishtar_common.Import']"}), + 'merge_candidate': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'merge_candidate_rel_+'", 'null': 'True', 'to': "orm['ishtar_common.Organization']"}), + 'merge_exclusion': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'merge_exclusion_rel_+'", 'null': 'True', 'to': "orm['ishtar_common.Organization']"}), + 'merge_key': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}), + 'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '300'}), + 'organization_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.OrganizationType']"}), + 'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}), + 'town': ('django.db.models.fields.CharField', [], {'max_length': '70', 'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.organizationtype': { + 'Meta': {'ordering': "('label',)", 'object_name': 'OrganizationType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}) + }, + 'ishtar_common.person': { + 'Meta': {'object_name': 'Person'}, + 'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'attached_to': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'members'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['ishtar_common.Organization']"}), + 'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'history_creator': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['auth.User']"}), + 'history_modifier': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['auth.User']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'imports': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'imported_ishtar_common_person'", 'null': 'True', 'symmetrical': 'False', 'to': "orm['ishtar_common.Import']"}), + 'merge_candidate': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'merge_candidate_rel_+'", 'null': 'True', 'to': "orm['ishtar_common.Person']"}), + 'merge_exclusion': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'merge_exclusion_rel_+'", 'null': 'True', 'to': "orm['ishtar_common.Person']"}), + 'merge_key': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}), + 'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'person_types': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['ishtar_common.PersonType']", 'symmetrical': 'False'}), + 'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}), + 'raw_name': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}), + 'surname': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'town': ('django.db.models.fields.CharField', [], {'max_length': '70', 'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.persontype': { + 'Meta': {'ordering': "('label',)", 'object_name': 'PersonType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': "orm['auth.Group']", 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}) + }, + 'ishtar_common.regexp': { + 'Meta': {'object_name': 'Regexp'}, + 'description': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'regexp': ('django.db.models.fields.CharField', [], {'max_length': '500'}) + }, + 'ishtar_common.sourcetype': { + 'Meta': {'object_name': 'SourceType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}) + }, + 'ishtar_common.state': { + 'Meta': {'ordering': "['number']", 'object_name': 'State'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '30'}), + 'number': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '3'}) + }, + 'ishtar_common.supporttype': { + 'Meta': {'object_name': 'SupportType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}) + }, + 'ishtar_common.targetkey': { + 'Meta': {'unique_together': "(('target', 'key', 'associated_user', 'associated_import'),)", 'object_name': 'TargetKey'}, + 'associated_import': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Import']", 'null': 'True', 'blank': 'True'}), + 'associated_user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.IshtarUser']", 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_set': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'key': ('django.db.models.fields.TextField', [], {}), + 'target': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'keys'", 'to': "orm['ishtar_common.ImportTarget']"}), + 'value': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.town': { + 'Meta': {'ordering': "['numero_insee']", 'object_name': 'Town'}, + 'canton': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Canton']", 'null': 'True', 'blank': 'True'}), + 'center': ('django.contrib.gis.db.models.fields.PointField', [], {'srid': '27572', 'null': 'True', 'blank': 'True'}), + 'departement': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Department']", 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'numero_insee': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '6'}), + 'surface': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}) + } + } + + complete_apps = ['ishtar_common']
\ No newline at end of file diff --git a/ishtar_common/models.py b/ishtar_common/models.py index 63ea7d2a8..fd0de385e 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -1149,6 +1149,8 @@ class ImporterType(models.Model): associated_models = models.CharField(_(u"Associated model"), max_length=200, choices=MODELS) is_template = models.BooleanField(_(u"Is template"), default=False) + unicity_keys = models.CharField(_(u"Unicity keys (separator \";\")"), + blank=True, null=True, max_length=500) class Meta: verbose_name = _(u"Importer - Type") @@ -1197,8 +1199,12 @@ class ImporterType(models.Model): formater = ImportFormater(targets, formater_types, **formater_kwargs) LINE_FORMAT.append(formater) + UNICITY_KEYS = [] + if self.unicity_keys: + UNICITY_KEYS = [un.strip() for un in self.unicity_keys.split(';')] args = {'OBJECT_CLS': OBJECT_CLS, 'DESC': self.description, - 'DEFAULTS': DEFAULTS, 'LINE_FORMAT': LINE_FORMAT} + 'DEFAULTS': DEFAULTS, 'LINE_FORMAT': LINE_FORMAT, + 'UNICITY_KEYS': UNICITY_KEYS} name = str(''.join( x for x in slugify(self.name).replace('-', ' ').title() if not x.isspace())) @@ -1355,6 +1361,8 @@ class ImportTarget(models.Model): formater_type = models.ForeignKey("FormaterType") force_new = models.BooleanField(_(u"Force creation of new item"), default=False) + concat = models.BooleanField(_(u"Concatenate with existing"), + default=False) comment = models.TextField(_(u"Comment"), blank=True, null=True) class Meta: @@ -1604,6 +1612,9 @@ class Import(models.Model): blank=True, null=True) state = models.CharField(_(u"State"), max_length=2, choices=IMPORT_STATE, default='C') + conservative_import = models.BooleanField( + _(u"Conservative import"), default=False, + help_text='If set to true, do not overload existing values') creation_date = models.DateTimeField(_(u"Creation date"), auto_now_add=True, blank=True, null=True) @@ -1656,7 +1667,8 @@ class Import(models.Model): def get_importer_instance(self): return self.importer_type.get_importer_class()( - skip_lines=self.skip_lines, import_instance=self) + skip_lines=self.skip_lines, import_instance=self, + conservative_import=self.conservative_import) @property def data_table(self): diff --git a/ishtar_common/wizards.py b/ishtar_common/wizards.py index abbdda711..c8b017293 100644 --- a/ishtar_common/wizards.py +++ b/ishtar_common/wizards.py @@ -677,7 +677,7 @@ class Wizard(NamedUrlWizardView): if not multi: key = key.startswith(form_key) and key or form_key + '-' + key val = request.session[storage.prefix]['step_data'][form_key][key] - if type(val) in (list, tuple): + if type(val) in (list, tuple) and val: val = val[0] return val vals = [] diff --git a/misc/pre_import_sra_files.py b/misc/pre_import_sra_files.py new file mode 100644 index 000000000..df00d3ef5 --- /dev/null +++ b/misc/pre_import_sra_files.py @@ -0,0 +1,79 @@ +import unicodecsv +import datetime + +from django.conf import settings + +from ishtar_common.data_importer import Importer + + +def get_year(value): + try: + for fmt in ['%d/%m/%Y', '%d/%m/%Y']: + return datetime.datetime.strptime(value, fmt).year + except: + pass + +index_list = [] + + +def treatment(data): + internal_ref = data[37].strip() + creation = data[34].strip() + reception = data[19].strip() + yr = get_year(creation) + if not yr: + yr = get_year(reception) + + idx, year = None, None + if '-' in internal_ref: + year, y_idx = internal_ref.split('-') + if len(year) == 4: # 2007-XXXX + try: + year = int(year) + idx = int(y_idx) + except ValueError: + pass + elif '.' in internal_ref: + year, y_idx = internal_ref.split('.') + if len(year) == 4: # 2011.XXXX + try: + year = int(year) + idx = int(y_idx) + except ValueError: + pass + if not idx: + idx = int(internal_ref) + if year and year != yr: + yr = year + assert yr # we should absolutly have a year! + + external_id = "{}{}-{}".format(settings.ISHTAR_LOCAL_PREFIX, yr, idx) + assert (yr, external_id) not in index_list + index_list.append((yr, external_id)) + return yr, idx, external_id + + +new_datas = [] +with open('plouf.csv') as csv_file: + datas = [line for line in unicodecsv.reader(csv_file, + encoding='utf-8')] + for idx, data in enumerate(datas): + if idx < 3: + # headers + data.append('annee') + data.append('identifiant numerique') + data.append('external_id') + new_datas.append(data) + continue + try: + year, idx, external_id = treatment(data) + data.append(year) + data.append(idx) + data.append(external_id) + new_datas.append(data) + except Exception as e: + print("Line {}: {}".format(idx + 1, e)) + +csv = Importer()._get_csv(new_datas, empty=u'') +with open('plouf2.csv', 'w') as fle: + fle.write(csv.encode('utf-8')) |