diff options
| -rw-r--r-- | archaeological_files/data_importer.py | 315 | ||||
| -rw-r--r-- | archaeological_files/migrations/0018_auto__add_field_file_imported_line__chg_field_file_responsible_town_pl.py | 304 | ||||
| -rw-r--r-- | archaeological_files/models.py | 1 | ||||
| -rw-r--r-- | archaeological_files/tests.py | 6 | ||||
| -rw-r--r-- | ishtar_common/data_importer.py | 541 | ||||
| -rw-r--r-- | ishtar_common/migrations/0012_auto__add_field_person_raw_name__chg_field_person_name.py | 210 | ||||
| -rw-r--r-- | ishtar_common/models.py | 7 |
7 files changed, 1208 insertions, 176 deletions
diff --git a/archaeological_files/data_importer.py b/archaeological_files/data_importer.py index b5f63fb67..23e9c6a32 100644 --- a/archaeological_files/data_importer.py +++ b/archaeological_files/data_importer.py @@ -17,11 +17,13 @@ # See the file COPYING for details. -import re, copy +import copy, datetime, re import unicodecsv from django.conf import settings +from django.db import IntegrityError from django.template.defaultfilters import slugify +from django.utils.translation import ugettext_lazy as _ from ishtar_common.data_importer import * from ishtar_common.models import Town, Person, OrganizationType @@ -29,7 +31,95 @@ from ishtar_common.unicode_csv import unicode_csv_reader from archaeological_files import models +from archaeological_operations.models import Parcel +from archaeological_operations.utils import parse_parcels + RE_FILTER_CEDEX = re.compile("(.*) *(?: *CEDEX|cedex|Cedex|Cédex|cédex *\d*)") +RE_PERMIT_REFERENCE = re.compile('[A-Za-z]*(.*)') + +class StrToBoolean(Formater): + def __init__(self, choices={}, cli=False, strict=False): + self.dct = copy.copy(choices) + self.cli = cli + self.strict= strict + self.missings = set() + + def prepare(self, value): + value = unicode(value).strip() + if not self.strict: + value = slugify(value) + return value + + def check(self, values): + msgstr = unicode(_(u"Choice for \"%s\" is not available. "\ + u"Which one is relevant?\n")) + msgstr += u"1. True\n" + msgstr += u"2. False\n" + msgstr += u"3. Empty\n" + for value in values: + value = self.prepare(value) + if value in self.dct: + continue + if not self.cli: + self.missings.add(value) + continue + res = None + while res not in range(1, 4): + sys.stdout.write(msgstr % value) + res = raw_input(">>> ") + try: + res = int(res) + except ValueError: + pass + if res == 1: + self.dct[value] = True + elif res == 2: + self.dct[value] = False + else: + self.dct[value] = None + + def format(self, value): + value = self.prepare(value) + if value in self.dct: + return self.dct[value] + +class ImportClosingFormater(ImportFormater): + def post_process(self, obj, context, value, owner=None): + value = self.formater.format(value) + if not value: + return + open_date = obj.reception_date or obj.creation_date + if not open_date: + return + obj.end_date = open_date + datetime.timedelta(30) + obj.save() + +class ImportParcelFormater(ImportFormater): + NEED = ['town',] + PARCEL_OWNER_KEY = 'associated_file' + + def post_process(self, obj, context, value, owner=None): + value = value.strip() + base_dct = {self.PARCEL_OWNER_KEY:obj, 'history_modifier':owner} + if 'parcels' in context: + for key in context['parcels']: + if context['parcels'][key]: + base_dct[key] = context['parcels'][key] + for parcel_dct in parse_parcels(value, owner=owner): + parcel_dct.update(base_dct) + try: + Parcel.objects.get_or_create(**parcel_dct) + except IntegrityError: + raise ImporterError("Erreur d'import parcelle, contexte : %s" \ + % unicode(parcel_dct)) + +class ImportYearFormater(ImportFormater): + def post_process(self, obj, context, value, owner=None): + value = self.formater.format(value) + if not value: + return + obj.year = value.year + obj.save() class TownFormater(Formater): def __init__(self, town_full_dct={}, town_dct={}): @@ -69,15 +159,56 @@ class TownFormater(Formater): if key in self._town_dct: return self._town_dct[key] +class TownINSEEFormater(Formater): + def __init__(self): + self._town_dct = {} + + def format(self, value, extra=None): + value = value.strip() + if not value: + return None + if value in self._town_dct: + return self._town_dct[value] + q = Town.objects.filter(insee_code=value) + if not q.count(): + return + self._town_dct[value] = q.all()[0] + return self._town_dct[value] + +class SurfaceFormater(Formater): + def test(self): + assert self.format(u"352 123") == 352123 + assert self.format(u"456 789 m²") == 456789 + assert self.format(u"78ha") == 780000 + + def format(self, value, extra=None): + value = value.strip() + if not value: + return None + factor = 1 + if value.endswith(u"m2") or value.endswith(u"m²"): + value = value[:-2] + if value.endswith(u"ha"): + value = value[:-2] + factor = 10000 + try: + return int(value.replace(' ', '')) * factor + except ValueError: + raise ImporterError("Erreur import surface : %s" \ + % unicode(value)) + #RE_ADD_CD_POSTAL_TOWN = re.compile("(.*)[, ](\d{5}) (.*?) *(?: "\ # "*CEDEX|cedex|Cedex *\d*)*") -RE_ADD_CD_POSTAL_TOWN = re.compile("(.*)?[, ]+(\d{5})[, ]+(.+)") +RE_NAME_ADD_CD_POSTAL_TOWN = re.compile("(.*)?[, ]*" + NEW_LINE_BREAK \ + + "(.*)?[, ]*(\d{2} *\d{3})[, ]*(.+)") RE_ADD_CD_POSTAL_TOWN = re.compile("(.*)?[, ]*(\d{2} *\d{3})[, ]*(.+)") RE_CD_POSTAL_FILTER = re.compile("(\d*) (\d*)") +RE_ORGA = re.compile("([^,]*)") + class FileImporterSraPdL(Importer): LINE_FORMAT = [] OBJECT_CLS = models.File @@ -89,30 +220,29 @@ class FileImporterSraPdL(Importer): txt_idx="general_contractor")}, tuple():{ 'file_type': models.FileType.objects.get( - txt_idx='undefined'),} + txt_idx='undefined'), + }, + ('in_charge',):{'attached_to':None}, # initialized in __init__ } def _init_line_format(self): tf = TownFormater() tf.town_dct_init() self.line_format = [ - ImportFormater('responsible_town_planning_service__name', - UnicodeFormater(300), - comment=u"Service instructeur - nom", - required=False), - ImportFormater(['address', 'postal_code', 'towns'], + None, # A, 1 + ImportFormater(['address', 'postal_code', ['towns', 'parcels__town']], # B, 2 [UnicodeFormater(500, clean=True), UnicodeFormater(5, re_filter=RE_CD_POSTAL_FILTER), tf], regexp=RE_ADD_CD_POSTAL_TOWN, regexp_formater_args=[[0], [1], [2, 1]], required=False, comment="Dossier - adresse"), - ImportFormater('general_contractor__name', + ImportFormater('general_contractor__raw_name', # C, 3 TODO - extraire nom_prenom_titre UnicodeFormater(200), - comment=u"Aménageur - nom", + comment=u"Aménageur - nom brut", duplicate_field='general_contractor__attached_to__name', required=False), - ImportFormater(['general_contractor__attached_to__address', + ImportFormater(['general_contractor__attached_to__address', # D, 4 'general_contractor__attached_to__postal_code', 'general_contractor__attached_to__town'], [UnicodeFormater(500, clean=True), @@ -122,13 +252,174 @@ class FileImporterSraPdL(Importer): regexp=RE_ADD_CD_POSTAL_TOWN, regexp_formater_args=[[0], [1], [2, 1]], required=False, comment="Aménageur - adresse"), - ImportFormater("general_contractor__title", + ImportFormater("general_contractor__title", # E, 5 StrChoiceFormater(Person.TYPE, cli=True), + required=False, + comment="Aménageur - titre"), + None, # F, 6 + None, # G, 7 + None, # H, 8 + ImportFormater("parcels__year", # I, 9 + YearNoFuturFormater(), required=False), + ImportParcelFormater('', required=False, post_processing=True), # J, 10 + None, # K, 11 + ImportFormater([['towns', 'parcels__town']], # L, 12 + tf, + required=False, + comment="Commune (si non définie avant)"), + ImportFormater([['towns', 'parcels__town']], # M, 13 + tf, + required=False, + comment="Commune (si non définie avant)"), + ImportFormater('saisine_type', # N, 14 + StrChoiceFormater(models.SaisineType.get_types(), + model=models.SaisineType, cli=True), + required=False, + comment="Type de saisine"), + None, # O, 15 + ImportFormater('comment', # P, 16 + UnicodeFormater(2000), + comment=u"Commentaire", + concat=True, required=False), + None, # Q, 17 + ImportFormater([ + 'responsible_town_planning_service__raw_name', # R, 18 service instructeur + 'responsible_town_planning_service__attached_to__address', + 'responsible_town_planning_service__attached_to__postal_code', + 'responsible_town_planning_service__attached_to__town',], + [UnicodeFormater(300, clean=True), + UnicodeFormater(300, clean=True), + UnicodeFormater(5, re_filter=RE_CD_POSTAL_FILTER), + TownFormater(town_full_dct=tf._town_full_dct, + town_dct=tf._town_dct)], + regexp=RE_NAME_ADD_CD_POSTAL_TOWN, + regexp_formater_args=[[0], [1], [2], [3, 2]], + comment="Aménageur - adresse", + required=False), + ImportFormater('comment', # S, 19 + UnicodeFormater(2000), + comment=u"Commentaire", + concat=True, required=False), + ImportYearFormater('reception_date', # T, 20 + DateFormater(), + comment=u"Date de création", + required=False, + duplicate_field='creation_date'), + None, # U, 21 + None, # V, 22 + None, # W, 23 + None, # X, 24 + None, # Y, 25 + None, # Z, 26 + None, # AA, 27 + None, # AB, 28 + None, # AC, 29 + None, # AD, 30 + None, # AE, 31 + None, # AF, 32 + None, # AG, 33 + None, # AH, 34 + ImportFormater('creation_date', # AI, 35 + DateFormater(), + force_value=True, + comment=u"Date de création", + required=False,), + None, # AJ, 36 + ImportFormater('comment', # AK, 37 + UnicodeFormater(2000), + comment=u"Commentaire", + concat=True, required=False), + None, # AL, 38 + None, # AM, 39 + None, # AN, 40 + None, # AO, 41 + ImportFormater('comment', # AP, 42 + UnicodeFormater(2000), + comment=u"Commentaire", + concat=True, required=False), + None, # AQ, 43 + None, # AR, 44 + None, # AS, 45 + None, # AT, 46 + ImportFormater('comment', # AU, 47 + UnicodeFormater(2000), + comment=u"Commentaire", + concat=True, required=False), + None, # AV, 48 + ImportFormater('permit_reference', # AW, 49 + UnicodeFormater(300, clean=True), + regexp=RE_PERMIT_REFERENCE, + comment="Réf. du permis de construire", + required=False), + None, # AX, 50 + None, # AY, 51 + None, # AZ, 52 + None, # BA, 53 + None, # BB, 54 + None, # BC, 55 + None, # BD, 56 + ImportFormater([['towns', 'parcels__town']], # BE, 57 + TownINSEEFormater(), + required=False, + comment="Commune (si non définie avant)"), + ImportFormater('comment', # BF, 58 + UnicodeFormater(2000), + comment=u"Commentaire", + concat=True, required=False), + None, # BG, 59 + None, # BH, 60 + None, # BI, 61 + None, # BJ, 62 + None, # BK, 63 + None, # BL, 64 + None, # BM, 65 + None, # BN, 66 + None, # BO, 67 + None, # BP, 68 + None, # BQ, 69 + None, # BR, 70 + None, # BS, 71 + ImportFormater( + 'responsible_town_planning_service__attached_to__name', # BT, 72 service instructeur + UnicodeFormater(300, clean=True), + regexp=RE_ORGA, + comment="Service instructeur - nom", + required=False), + None, # BU, 73 + ImportClosingFormater('', StrToBoolean(cli=True), + post_processing=True, required=False), # BV, 74, end date + ImportClosingFormater('in_charge__raw_name', # BW, 75 responsable + UnicodeFormater(200), + comment=u"Responsable - nom brut", + required=False), + ImportFormater('total_surface', # BX, 76 surface totale + SurfaceFormater(), + comment=u"Surface totale", + required=False), + ImportFormater('total_developed_surface', # BY, 77 surface totale aménagée + SurfaceFormater(), + comment=u"Surface totale aménagée", + required=False), + None, # BZ, 78 + None, # CA, 79 + None, # CB, 80 + None, # CC, 81 + None, # CD, 82 + None, # CE, 83 + None, # CF, 84 + ImportFormater('permit_type', + StrChoiceFormater(models.PermitType.get_types(), + model=models.PermitType, cli=True), + required=False, + comment="Type de permis"), # CG, 85 + None, # CH, 85 ] def __init__(self, *args, **kwargs): super(FileImporterSraPdL, self).__init__(*args, **kwargs) + self.DEFAULTS[('in_charge',)]['attached_to'] = \ + models.Organization.objects.get(name='SRA Pays de la Loire') self._init_line_format() if tuple() not in self._defaults: self._defaults[tuple()] = {} diff --git a/archaeological_files/migrations/0018_auto__add_field_file_imported_line__chg_field_file_responsible_town_pl.py b/archaeological_files/migrations/0018_auto__add_field_file_imported_line__chg_field_file_responsible_town_pl.py new file mode 100644 index 000000000..4555145a3 --- /dev/null +++ b/archaeological_files/migrations/0018_auto__add_field_file_imported_line__chg_field_file_responsible_town_pl.py @@ -0,0 +1,304 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'File.imported_line' + db.add_column('archaeological_files_file', 'imported_line', + self.gf('django.db.models.fields.TextField')(null=True, blank=True), + keep_default=False) + + + # Changing field 'File.responsible_town_planning_service' + db.alter_column('archaeological_files_file', 'responsible_town_planning_service_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, on_delete=models.SET_NULL, to=orm['ishtar_common.Person'])) + + # Changing field 'File.general_contractor' + db.alter_column('archaeological_files_file', 'general_contractor_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, on_delete=models.SET_NULL, to=orm['ishtar_common.Person'])) + + # Changing field 'File.in_charge' + db.alter_column('archaeological_files_file', 'in_charge_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, on_delete=models.SET_NULL, to=orm['ishtar_common.Person'])) + + # Changing field 'File.scientist' + db.alter_column('archaeological_files_file', 'scientist_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, on_delete=models.SET_NULL, to=orm['ishtar_common.Person'])) + + # Changing field 'File.organization' + db.alter_column('archaeological_files_file', 'organization_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, on_delete=models.SET_NULL, to=orm['ishtar_common.Organization'])) + # Adding field 'HistoricalFile.imported_line' + db.add_column('archaeological_files_historicalfile', 'imported_line', + self.gf('django.db.models.fields.TextField')(null=True, blank=True), + keep_default=False) + + + def backwards(self, orm): + # Deleting field 'File.imported_line' + db.delete_column('archaeological_files_file', 'imported_line') + + + # Changing field 'File.responsible_town_planning_service' + db.alter_column('archaeological_files_file', 'responsible_town_planning_service_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, to=orm['ishtar_common.Person'])) + + # Changing field 'File.general_contractor' + db.alter_column('archaeological_files_file', 'general_contractor_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, to=orm['ishtar_common.Person'])) + + # Changing field 'File.in_charge' + db.alter_column('archaeological_files_file', 'in_charge_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, to=orm['ishtar_common.Person'])) + + # Changing field 'File.scientist' + db.alter_column('archaeological_files_file', 'scientist_id', self.gf('django.db.models.fields.related.ForeignKey')(null=True, to=orm['ishtar_common.Person'])) + + # Changing field 'File.organization' + db.alter_column('archaeological_files_file', 'organization_id', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['ishtar_common.Organization'], null=True)) + # Deleting field 'HistoricalFile.imported_line' + db.delete_column('archaeological_files_historicalfile', 'imported_line') + + + models = { + 'archaeological_files.file': { + 'Meta': {'ordering': "('cached_label',)", 'object_name': 'File'}, + 'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'cached_label': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}), + 'cira_advised': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'classified_area': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'creation_date': ('django.db.models.fields.DateField', [], {'default': 'datetime.date.today'}), + 'departments': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': "orm['ishtar_common.Department']", 'null': 'True', 'blank': 'True'}), + 'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'file_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['archaeological_files.FileType']"}), + 'general_contractor': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'general_contractor'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['ishtar_common.Person']"}), + 'history_creator': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'to': "orm['auth.User']"}), + 'history_modifier': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['auth.User']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'imported_line': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'in_charge': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'file_responsability'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['ishtar_common.Person']"}), + 'internal_reference': ('django.db.models.fields.CharField', [], {'max_length': '60', 'null': 'True', 'blank': 'True'}), + 'mh_listing': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'mh_register': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'numeric_reference': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'organization': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'files'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['ishtar_common.Organization']"}), + 'permit_reference': ('django.db.models.fields.CharField', [], {'max_length': '60', 'null': 'True', 'blank': 'True'}), + 'permit_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['archaeological_files.PermitType']", 'null': 'True', 'blank': 'True'}), + 'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}), + 'protected_area': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'reception_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'reference_number': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'related_file': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['archaeological_files.File']", 'null': 'True', 'blank': 'True'}), + 'requested_operation_type': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'to': "orm['archaeological_operations.OperationType']"}), + 'research_comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'responsible_town_planning_service': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'responsible_town_planning_service'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['ishtar_common.Person']"}), + 'saisine_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['archaeological_files.SaisineType']", 'null': 'True', 'blank': 'True'}), + 'scientist': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'scientist'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['ishtar_common.Person']"}), + 'total_developed_surface': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'total_surface': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'towns': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'file'", 'symmetrical': 'False', 'to': "orm['ishtar_common.Town']"}), + 'year': ('django.db.models.fields.IntegerField', [], {'default': '2014'}) + }, + 'archaeological_files.filebydepartment': { + 'Meta': {'object_name': 'FileByDepartment', 'db_table': "'file_department'", 'managed': 'False'}, + 'department': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Department']", 'null': 'True', 'blank': 'True'}), + 'file': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['archaeological_files.File']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + 'archaeological_files.filetype': { + 'Meta': {'ordering': "('label',)", 'object_name': 'FileType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'archaeological_files.historicalfile': { + 'Meta': {'ordering': "('-history_date', '-history_id')", 'object_name': 'HistoricalFile'}, + 'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'cached_label': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}), + 'cira_advised': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'classified_area': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'creation_date': ('django.db.models.fields.DateField', [], {'default': 'datetime.date.today'}), + 'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'file_type_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'general_contractor_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'history_creator_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'history_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'history_id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'history_modifier_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'history_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'history_user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True'}), + 'id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'blank': 'True'}), + 'imported_line': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'in_charge_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'internal_reference': ('django.db.models.fields.CharField', [], {'max_length': '60', 'null': 'True', 'blank': 'True'}), + 'mh_listing': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'mh_register': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'numeric_reference': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'organization_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'permit_reference': ('django.db.models.fields.CharField', [], {'max_length': '60', 'null': 'True', 'blank': 'True'}), + 'permit_type_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}), + 'protected_area': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'reception_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'reference_number': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'related_file_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'requested_operation_type_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'research_comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'responsible_town_planning_service_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'saisine_type_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'scientist_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'total_developed_surface': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'total_surface': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'year': ('django.db.models.fields.IntegerField', [], {'default': '2014'}) + }, + 'archaeological_files.permittype': { + 'Meta': {'ordering': "('label',)", 'object_name': 'PermitType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'archaeological_files.saisinetype': { + 'Meta': {'ordering': "('label',)", 'object_name': 'SaisineType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'delay': ('django.db.models.fields.IntegerField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'archaeological_operations.operationtype': { + 'Meta': {'ordering': "['-preventive', 'order', 'label']", 'object_name': 'OperationType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'order': ('django.db.models.fields.IntegerField', [], {'default': '1'}), + 'preventive': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'ishtar_common.arrondissement': { + 'Meta': {'object_name': 'Arrondissement'}, + 'department': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Department']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '30'}) + }, + 'ishtar_common.canton': { + 'Meta': {'object_name': 'Canton'}, + 'arrondissement': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Arrondissement']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '30'}) + }, + 'ishtar_common.department': { + 'Meta': {'ordering': "['number']", 'object_name': 'Department'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '30'}), + 'number': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '3'}) + }, + 'ishtar_common.organization': { + 'Meta': {'object_name': 'Organization'}, + 'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'history_creator': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'to': "orm['auth.User']"}), + 'history_modifier': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['auth.User']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '300'}), + 'organization_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.OrganizationType']"}), + 'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}), + 'town': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.organizationtype': { + 'Meta': {'ordering': "('label',)", 'object_name': 'OrganizationType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'ishtar_common.person': { + 'Meta': {'object_name': 'Person'}, + 'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'attached_to': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'members'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['ishtar_common.Organization']"}), + 'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'history_creator': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'to': "orm['auth.User']"}), + 'history_modifier': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['auth.User']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}), + 'person_types': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['ishtar_common.PersonType']", 'symmetrical': 'False'}), + 'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}), + 'surname': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '2'}), + 'town': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.persontype': { + 'Meta': {'ordering': "('label',)", 'object_name': 'PersonType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': "orm['auth.Group']", 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'ishtar_common.town': { + 'Meta': {'ordering': "['numero_insee']", 'object_name': 'Town'}, + 'canton': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Canton']", 'null': 'True', 'blank': 'True'}), + 'center': ('django.contrib.gis.db.models.fields.PointField', [], {'srid': '27572', 'null': 'True', 'blank': 'True'}), + 'departement': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Department']", 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'numero_insee': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '6'}), + 'surface': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}) + } + } + + complete_apps = ['archaeological_files']
\ No newline at end of file diff --git a/archaeological_files/models.py b/archaeological_files/models.py index 0c18af090..a1b42f722 100644 --- a/archaeological_files/models.py +++ b/archaeological_files/models.py @@ -147,6 +147,7 @@ class File(BaseHistorizedItem, OwnPerms, ValueGetter, ShortMenuItem, # <-- research archaeology cached_label = models.CharField(_(u"Cached name"), max_length=500, null=True, blank=True) + imported_line = models.TextField(_(u"Imported line"), null=True, blank=True) history = HistoricalRecords() class Meta: diff --git a/archaeological_files/tests.py b/archaeological_files/tests.py index b43967401..cf73b8726 100644 --- a/archaeological_files/tests.py +++ b/archaeological_files/tests.py @@ -27,7 +27,7 @@ from django.contrib.auth.models import User from django.test import TestCase from ishtar_common.models import PersonType -import models +from archaeological_files import models, data_importer class FileTest(TestCase): fixtures = [settings.ROOT_PATH + \ @@ -152,3 +152,7 @@ class FileTest(TestCase): self.assertTrue(data['records'] == 1) self.assertEqual(data['rows'][0]['internal_reference'], initial_ref) +class ImporterTest(TestCase): + def testFormaters(self): + for formater in [data_importer.SurfaceFormater]: + formater().test() diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index 86285e33e..1d768c6b0 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -17,139 +17,24 @@ # See the file COPYING for details. -""" -# Usage exemple (extracted from simulabio application) - -class HarvestPlotImporter(Importer): - LINE_FORMAT = [ - ImportFormater('name', Importer.get_unicode_formater(100)), - ImportFormater('plot_group_number', - Importer.get_unicode_formater(3), required=False), - ImportFormater('geographical_area', unicode, required=False), - ImportFormater('soil_type', Importer.choices_check(SOIL_TYPE), - required=False), - ImportFormater('cow_access', Importer.boolean_formater), - ImportFormater('area', Importer.float_formater), - ImportFormater('remark', unicode, required=False), - ImportFormater('diagnostic', Importer.boolean_formater), - ImportFormater('project', Importer.boolean_formater), - ImportFormater('harvest_n2', 'harvest_formater', required=False), - ImportFormater('harvest_n1', 'harvest_formater', required=False), - ImportFormater('harvest', 'harvest_formater'), - ImportFormater('harvest_setting', 'harvest_formater', - through=HarvestTransition, - through_key='plot', - through_dict={'year':1}, - through_unicity_keys=['plot', 'year'], - required=False), - ImportFormater('harvest_setting', 'harvest_formater', - through=HarvestTransition, - through_key='plot', - through_dict={'year':2}, - through_unicity_keys=['plot', 'year'], - required=False), - ImportFormater('harvest_setting', 'harvest_formater', - through=HarvestTransition, - through_key='plot', - through_dict={'year':3}, - through_unicity_keys=['plot', 'year'], - required=False), - ImportFormater('harvest_setting', 'harvest_formater', - through=HarvestTransition, - through_key='plot', - through_dict={'year':4}, - through_unicity_keys=['plot', 'year'], - required=False), - ImportFormater('harvest_setting', 'harvest_formater', - through=HarvestTransition, - through_key='plot', - through_dict={'year':5}, - through_unicity_keys=['plot', 'year'], - required=False), - ImportFormater('harvest_setting', 'harvest_formater', - through=HarvestTransition, - through_key='plot', - through_dict={'year':6}, - through_unicity_keys=['plot', 'year'], - required=False), - ] - OBJECT_CLS = HarvestPlots - UNICITY_KEYS = [] - - def __init__(self, study, skip_first_line=None): - # get the reference header - dct = {'separator':settings.CSV_DELIMITER} - dct['data'] = Harvest.objects.filter(available=True).all() - reference_file = render_to_string('simulabio/files/parcelles_ref.csv', - dct) - reference_header = unicode_csv_reader( - [reference_file.split('\n')[0]]).next() - super(HarvestPlotImporter, self).__init__( - skip_first_line=skip_first_line, - reference_header=reference_header) - self.study = study - self.default_vals = {'study':self.study} - - def harvest_formater(self, value): - value = value.strip() - if not value: - return - try: - harvest = Harvest.objects.get(name__iexact=value) - except ObjectDoesNotExist: - raise ValueError(_(u"\"%(value)s\" not in %(values)s") % { - 'value':value, - 'values':u", ".join([val.name - for val in Harvest.objects.filter(available=True)]) - }) - hs, created = HarvestSettings.objects.get_or_create(study=self.study, - harvest=harvest) - if created: - self.message = _(u"\"%(harvest)s\" has been added in your settings " - u"don't forget to fill yields for this harvest.") \ - % {'harvest':harvest.name} - return hs - -class HarvestPlotsImportForm(forms.Form): - csv_file = forms.FileField(label=_(u"Plot list file (CSV)")) - - def save(self, study): - csv_file = self.cleaned_data['csv_file'] - importer = models.HarvestPlotImporter(study, skip_first_line=True) - # some softwares (at least Gnumeric) convert CSV file to utf-8 no matter - # what the CSV source encoding is - encodings = [settings.ENCODING, 'utf-8'] - for encoding in encodings: - try: - importer.importation(unicode_csv_reader( - [line.decode(encoding) - for line in csv_file.readlines()])) - except ImporterError, e: - if e.type == ImporterError.HEADER and encoding != encodings[-1]: - csv_file.seek(0) - continue - return 0, [[0, 0, e.msg]], [] - except UnicodeDecodeError, e: - return 0, [[0, 0, Importer.ERRORS['header_check']]], [] - break - return importer.number_imported, importer.errors, importer.messages -""" - import copy, csv, datetime, logging, sys from tempfile import NamedTemporaryFile from django.contrib.auth.models import User -from django.db import DatabaseError +from django.db import DatabaseError, IntegrityError from django.template.defaultfilters import slugify from django.utils.translation import ugettext_lazy as _ from ishtar_common.unicode_csv import UnicodeWriter +NEW_LINE_BREAK = '#####@@@#####' + class ImportFormater(object): def __init__(self, field_name, formater=None, required=True, through=None, through_key=None, through_dict=None, through_unicity_keys=None, duplicate_field=None, regexp=None, regexp_formater_args=[], - reverse_for_test=None, comment=""): + reverse_for_test=None, force_value=None, post_processing=False, + concat=False, comment=""): self.field_name = field_name self.formater = formater self.required = required @@ -161,6 +46,12 @@ class ImportFormater(object): self.regexp = regexp self.regexp_formater_args = regexp_formater_args self.reverse_for_test = reverse_for_test + # write this value even if a value exists + self.force_value = force_value + # post process after import + self.post_processing = post_processing + # concatenate with existing value + self.concat = concat self.comment = comment def __unicode__(self): @@ -178,7 +69,11 @@ class ImportFormater(object): except TypeError: lst = [self.formater] for formater in lst: - formater.check(vals) + if formater: + formater.check(vals) + + def post_process(self, obj, context, value, owner=None): + raise NotImplemented() class ImporterError(Exception): STANDARD = 'S' @@ -186,6 +81,7 @@ class ImporterError(Exception): def __init__(self, message, type='S'): self.msg = message self.type = type + def __str__(self): return self.msg @@ -214,6 +110,7 @@ class UnicodeFormater(Formater): value = value[1:] if value.endswith(","): value = value[:-1] + value = value.replace(", , ", ", ") except UnicodeDecodeError: return if len(value) > self.max_length: @@ -243,19 +140,59 @@ class FloatFormater(Formater): raise ValueError(_(u"\"%(value)s\" is not a float") % { 'value':value}) +class YearFormater(Formater): + def format(self, value): + value = value.strip() + if not value: + return + try: + value = int(value) + assert value > 0 and value < (datetime.date.today().year + 30) + except (ValueError, AssertionError): + raise ValueError(_(u"\"%(value)s\" is not a valid date") % { + 'value':value}) + +class YearNoFuturFormater(Formater): + def format(self, value): + value = value.strip() + if not value: + return + try: + value = int(value) + assert value > 0 and value < (datetime.date.today().year) + except (ValueError, AssertionError): + raise ValueError(_(u"\"%(value)s\" is not a valid date") % { + 'value':value}) + +class IntegerFormater(Formater): + def format(self, value): + value = value.strip() + if not value: + return + try: + return int(value) + except ValueError: + raise ValueError(_(u"\"%(value)s\" is not an integer") % { + 'value':value}) + class StrChoiceFormater(Formater): - def __init__(self, choices, strict=False, equiv_dict={}, cli=False): - self.choices = choices + def __init__(self, choices, strict=False, equiv_dict={}, model=None, + cli=False): + self.choices = list(choices) self.strict = strict self.equiv_dict = copy.deepcopy(equiv_dict) self.cli = cli + self.model = model self.missings = set() for key, value in self.choices: value = unicode(value) if not self.strict: value = slugify(value) if value not in self.equiv_dict: - self.equiv_dict[value] = key + v = key + if model and v: + v = model.objects.get(pk=v) + self.equiv_dict[value] = v def prepare(self, value): return unicode(value).strip() @@ -283,7 +220,10 @@ class StrChoiceFormater(Formater): pass res -= 1 if res < len(self.choices): - self.equiv_dict[value] = self.choices[res] + v = self.choices[res][0] + if self.model and v: + v = self.model.objects.get(pk=v) + self.equiv_dict[value] = v else: self.equiv_dict[value] = None @@ -294,11 +234,24 @@ class StrChoiceFormater(Formater): if value in self.equiv_dict: return self.equiv_dict[value] +class DateFormater(Formater): + def __init__(self, date_format="%d/%m/%Y"): + self.date_format = date_format + + def format(self, value): + value = value.strip() + try: + return datetime.datetime.strptime(value, self.date_format).date() + except: + raise ValueError(_(u"\"%(value)s\" is not a valid date") % { + 'value':value}) + logger = logging.getLogger(__name__) class Importer(object): LINE_FORMAT = [] OBJECT_CLS = None + IMPORTED_LINE_FIELD = None UNICITY_KEYS = [] DEFAULTS = {} ERRORS = { @@ -356,23 +309,264 @@ class Importer(object): vals.append([]) vals[idx_col].append(val) for idx, formater in enumerate(self.line_format): - formater.init(vals[idx]) + if formater: + formater.init(vals[idx]) self._initialized = True def importation(self, table): + self.validity_file = None if not self._initialized: self.initialize(table) if self.check_validity: with NamedTemporaryFile(delete=False) as validity_file: - print(validity_file.name) - validity_file_writer = UnicodeWriter(validity_file, + self.validity_file = UnicodeWriter(validity_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) - self._importation(table, validity_file_writer) + self._importation(table) else: self._importation(table) - def _importation(self, table, validity_file=None): + @classmethod + def _field_name_to_data_dict(cls, field_name, value, data, + force_value=False, concat=False): + field_names = field_name + if type(field_names) not in (list, tuple): + field_names = [field_name] + for field_name in field_names: + keys = field_name.split('__') + current_data = data + for idx, key in enumerate(keys): + if idx == (len(keys) - 1): # last + if concat: + if not value: + value = "" + current_data[key] = (current_data[key] + u"\n") or u""\ + + value + elif force_value and value: + current_data[key] = value + elif key not in current_data or not current_data[key]: + current_data[key] = value + elif key not in current_data: + current_data[key] = {} + current_data = current_data[key] + return data + + def _importation(self, table): + table = list(table) + if not table or not table[0]: + raise ImporterError(self.ERRORS['no_data'], ImporterError.HEADER) + if self.check_col_num and len(table[0]) > len(self.line_format): + raise ImporterError(self.ERRORS['too_many_cols'] % { + 'user_col':len(table[0]), 'ref_col':len(self.line_format)}) + self.errors = [] + self.messages = [] + self.number_imported = 0 + # index of the last required column + for idx_last_col, formater in enumerate(reversed(self.line_format)): + if formater and formater.required: + break + else: + idx_last_col += 1 + # min col number to be filled + self.min_col_number = len(self.line_format) - idx_last_col + # check the conformity with the reference header + if self.reference_header and \ + self.skip_first_line and \ + self.reference_header != table[0]: + raise ImporterError(self.ERRORS['header_check'], + type=ImporterError.HEADER) + self.now = datetime.datetime.now() + for idx_line, line in enumerate(table): + self._line_processing(idx_line, line) + + def _line_processing(self, idx_line, line): + if (self.skip_first_line and not idx_line): + if self.validity_file: + self.validity_file.writerow(line) + return + if not line: + if self.validity_file: + self.validity_file.writerow([]) + return + self._throughs = [] # list of (formater, value) + self._post_processing = [] # list of (formater, value) + data = {} + + # keep in database the raw line for testing purpose + if self.IMPORTED_LINE_FIELD: + output = io.StringIO() + writer = csv.writer(output) + writer.writerow(line) + data[self.IMPORTED_LINE_FIELD] = output.getvalue() + + n = datetime.datetime.now() + logger.debug('%s - Processing line %d' % (unicode(n-self.now), idx_line)) + self.now = n + n2 = n + self.c_errors = False + c_row = [] + for idx_col, val in enumerate(line): + try: + self._row_processing(c_row, idx_col, idx_line, val, data) + except: + pass + + if self.validity_file: + self.validity_file.writerow(c_row) + if not self.c_errors and (idx_col + 1) < self.min_col_number: + self.c_errors = True + self.errors.append((idx_line+1, idx_col+1, + self.ERRORS['not_enough_cols'] % self.min_col_number)) + if self.c_errors: + return + n = datetime.datetime.now() + logger.debug('* %s - Cols read' % (unicode(n-n2))) + n2 = n + if self.test: + return + # manage unicity of items (mainly for updates) + self.number_imported += 1 + if self.UNICITY_KEYS: + data['defaults'] = {} + for k in data.keys(): + if k not in self.UNICITY_KEYS \ + and k != 'defaults': + data['defaults'][k] = data.pop(k) + + data['history_modifier'] = self.history_modifier + obj, created = self.get_object(self.OBJECT_CLS, data) + + if not created and 'defaults' in data: + for k in data['defaults']: + setattr(obj, k, data['defaults'][k]) + obj.save() + n = datetime.datetime.now() + logger.debug('* %s - Item saved' % (unicode(n-n2))) + n2 = n + for formater, value in self._throughs: + n = datetime.datetime.now() + logger.debug('* %s - Processing formater %s' % (unicode(n-n2), + formater.field_name)) + n2 = n + data = {} + if formater.through_dict: + data = formater.through_dict.copy() + if formater.through_key: + data[formater.through_key] = obj + data[formater.field_name] = value + through_cls = formater.through + if formater.through_unicity_keys: + data['defaults'] = {} + for k in data.keys(): + if k not in formater.through_unicity_keys \ + and k != 'defaults': + data['defaults'][k] = data.pop(k) + t_obj, created = through_cls.objects.get_or_create(**data) + if not created and 'defaults' in data: + for k in data['defaults']: + setattr(t_obj, k, data['defaults'][k]) + t_obj.save() + + for formater, val in self._post_processing: + formater.post_process(obj, data, val, owner=self.history_modifier) + + def _row_processing(self, c_row, idx_col, idx_line, val, data): + if idx_col >= len(self.line_format): + return + + formater = self.line_format[idx_col] + + if formater and formater.post_processing: + self._post_processing.append((formater, val)) + + if not formater or not formater.field_name: + if self.validity_file: + c_row.append(val) + return + + # regex management + if formater.regexp: + # multiline regexp is a mess... + val = val.replace('\n', NEW_LINE_BREAK) + match = formater.regexp.match(val) + if not match: + if formater.required: + self.errors.append((idx_line+1, idx_col+1, + self.ERRORS['value_required'])) + elif not val.strip(): + c_row.append("") + return + self.c_errors = True + val = val.replace(NEW_LINE_BREAK, '\n') + self.errors.append((idx_line+1, idx_col+1, + unicode(self.ERRORS['regex_not_match']) + val)) + c_row.append("") + return + val_group = [v.replace(NEW_LINE_BREAK, '\n') + for v in match.groups()] + else: + val_group = [val] + + c_values = [] + for idx_v, v in enumerate(val_group): + self.message = '' + func = formater.formater + if type(func) in (list, tuple): + func = func[idx_v] + if not callable(func) and type(func) in (unicode, str): + func = getattr(self, func) + value = None + + try: + if formater.regexp_formater_args: + args = [] + for idx in formater.regexp_formater_args[idx_v]: + args.append(val_group[idx]) + value = func.format(*args) + else: + value = func.format(v) + except ValueError, e: + if formater.required: + self.c_errors = True + self.errors.append((idx_line+1, idx_col+1, e.message)) + c_values.append(None) + return + + if self.message: + self.messages.append(self.message) + c_values.append(value) + + if value == None: + if formater.required: + self.c_errors = True + self.errors.append((idx_line+1, idx_col+1, + self.ERRORS['value_required'])) + return + + field_name = formater.field_name + if type(field_name) in (list, tuple): + field_name = field_name[idx_v] + field_names = [field_name] + if formater.duplicate_field: + duplicate_field = formater.duplicate_field + if type(duplicate_field) in (list, tuple): + duplicate_field = duplicate_field[idx_v] + field_names += [duplicate_field] + + if formater.through: + self._throughs.append((formater, value)) + else: + for field_name in field_names: + self._field_name_to_data_dict(field_name, + value, data, formater.force_value) + if formater.reverse_for_test: + c_row.append(formater.reverse_for_test(**c_values)) + else: + c_row.append(unicode(c_values)) + + + """ + def _importation(self, table): table = list(table) if not table or not table[0]: raise ImporterError(self.ERRORS['no_data'], ImporterError.HEADER) @@ -398,33 +592,46 @@ class Importer(object): type=ImporterError.HEADER) now = datetime.datetime.now() for idx_line, line in enumerate(table): + #self._line_processing() + if (self.skip_first_line and not idx_line): - if validity_file: - validity_file.writerow(line) + if self.validity_file: + self.validity_file.writerow(line) continue if not line: - if validity_file: - validity_file.writerow([]) + if self.validity_file: + self.validity_file.writerow([]) continue - throughs = [] # list of (formater, value) + self.throughs = [] # list of (formater, value) + self.post_processing = [] # list of (formater, value) data = {} + + # keep in database the raw line for testing purpose + if self.IMPORTED_LINE_FIELD: + output = io.StringIO() + writer = csv.writer(output) + writer.writerow(line) + data[self.IMPORTED_LINE_FIELD] = output.getvalue() + n = datetime.datetime.now() logger.debug('%s - Processing line %d' % (unicode(n-now), idx_line)) now = n n2 = n - c_errors = False + self.c_errors = False c_row = [] for idx_col, val in enumerate(line): + #self._row_processing(self, c_row, idx_col, val): + if idx_col >= len(self.line_format): break formater = self.line_format[idx_col] if not formater.field_name: - if validity_file: + if self.validity_file: c_row.append(val) continue if formater.regexp: # multiline regexp is a mess... - val = val.replace('\n', '######???#####') + val = val.replace('\n', NEW_LINE_BREAK) match = formater.regexp.match(val) if not match: if formater.required: @@ -434,12 +641,12 @@ class Importer(object): c_row.append("") continue c_errors = True - val = val.replace('######???#####', '\n') + val = val.replace(NEW_LINE_BREAK, '\n') self.errors.append((idx_line+1, idx_col+1, unicode(self.ERRORS['regex_not_match']) + val)) c_row.append("") continue - val_group = [v.replace('######???#####', '\n') + val_group = [v.replace(NEW_LINE_BREAK, '\n') for v in match.groups()] else: val_group = [val] @@ -483,29 +690,26 @@ class Importer(object): if type(duplicate_field) in (list, tuple): duplicate_field = duplicate_field[idx_v] field_names += [duplicate_field] - if not formater.through: - for field_name in field_names: - keys = field_name.split('__') - current_data = data - for idx, key in enumerate(keys): - if idx == (len(keys) - 1): # last - current_data[key] = value - elif key not in current_data: - current_data[key] = {} - current_data = current_data[key] - else: + + + if formater.through: throughs.append((formater, value)) + else: + for field_name in field_names: + self._field_name_to_data_dict(field_name, + value, data) if formater.reverse_for_test: c_row.append(formater.reverse_for_test(**c_values)) else: c_row.append(unicode(c_values)) - if validity_file: - validity_file.writerow(c_row) - if not c_errors and (idx_col + 1) < min_col_number: - c_errors = True + + if self.validity_file: + self.validity_file.writerow(c_row) + if not self.c_errors and (idx_col + 1) < min_col_number: + self.c_errors = True self.errors.append((idx_line+1, idx_col+1, self.ERRORS['not_enough_cols'] % min_col_number)) - if c_errors: + if self.c_errors: continue n = datetime.datetime.now() logger.debug('* %s - Cols read' % (unicode(n-n2))) @@ -530,7 +734,7 @@ class Importer(object): n = datetime.datetime.now() logger.debug('* %s - Item saved' % (unicode(n-n2))) n2 = n - for formater, value in throughs: + for formater, value in self.throughs: n = datetime.datetime.now() logger.debug('* %s - Processing formater %s' % (unicode(n-n2), formater.field_name)) @@ -553,6 +757,7 @@ class Importer(object): for k in data['defaults']: setattr(t_obj, k, data['defaults'][k]) t_obj.save() + """ def get_object(self, cls, data, path=[]): m2ms = [] @@ -563,7 +768,8 @@ class Importer(object): continue field_object, model, direct, m2m = \ cls._meta.get_field_by_name(attribute) - if field_object.rel and type(data[attribute]) == dict: + if hasattr(field_object, 'rel') and field_object.rel and \ + type(data[attribute]) == dict: c_path.append(attribute) # put history_modifier for every created item data[attribute]['history_modifier'] = \ @@ -577,15 +783,26 @@ class Importer(object): for k in self._defaults[path]: if k not in data or not data[k]: data[k] = self._defaults[path][k] - obj, created = cls.objects.get_or_create(**data) - for attr, value in m2ms: - getattr(obj, attr).add(value) + + # filter default values + create_dict = copy.deepcopy(data) + for k in create_dict.keys(): + if type(create_dict[k]) == dict: + create_dict.pop(k) + + try: + obj, created = cls.objects.get_or_create(**create_dict) + for attr, value in m2ms: + getattr(obj, attr).add(value) + except IntegrityError: + raise ImporterError("Erreur d'import %s, contexte : %s" \ + % (unicode(cls), unicode(data))) return obj, created return data def get_csv_errors(self): for line, col, error in self.errors: - print '"%d","%d","%s"' % (line, col, unicode(error)) + print('"%d","%d","%s"' % (line, col, unicode(error))) @classmethod def choices_check(cls, choices): diff --git a/ishtar_common/migrations/0012_auto__add_field_person_raw_name__chg_field_person_name.py b/ishtar_common/migrations/0012_auto__add_field_person_raw_name__chg_field_person_name.py new file mode 100644 index 000000000..015956610 --- /dev/null +++ b/ishtar_common/migrations/0012_auto__add_field_person_raw_name__chg_field_person_name.py @@ -0,0 +1,210 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'Person.raw_name' + db.add_column('ishtar_common_person', 'raw_name', + self.gf('django.db.models.fields.CharField')(max_length=300, null=True, blank=True), + keep_default=False) + + + # Changing field 'Person.name' + db.alter_column('ishtar_common_person', 'name', self.gf('django.db.models.fields.CharField')(max_length=200, null=True)) + + def backwards(self, orm): + # Deleting field 'Person.raw_name' + db.delete_column('ishtar_common_person', 'raw_name') + + + # User chose to not deal with backwards NULL issues for 'Person.name' + raise RuntimeError("Cannot reverse this migration. 'Person.name' and its values cannot be restored.") + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'ishtar_common.arrondissement': { + 'Meta': {'object_name': 'Arrondissement'}, + 'department': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Department']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '30'}) + }, + 'ishtar_common.author': { + 'Meta': {'object_name': 'Author'}, + 'author_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.AuthorType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'person': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'author'", 'to': "orm['ishtar_common.Person']"}) + }, + 'ishtar_common.authortype': { + 'Meta': {'object_name': 'AuthorType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'ishtar_common.canton': { + 'Meta': {'object_name': 'Canton'}, + 'arrondissement': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Arrondissement']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '30'}) + }, + 'ishtar_common.department': { + 'Meta': {'ordering': "['number']", 'object_name': 'Department'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '30'}), + 'number': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '3'}) + }, + 'ishtar_common.documenttemplate': { + 'Meta': {'ordering': "['associated_object_name']", 'object_name': 'DocumentTemplate'}, + 'associated_object_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'template': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}) + }, + 'ishtar_common.globalvar': { + 'Meta': {'ordering': "['slug']", 'object_name': 'GlobalVar'}, + 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '50'}), + 'value': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.historicalorganization': { + 'Meta': {'ordering': "('-history_date', '-history_id')", 'object_name': 'HistoricalOrganization'}, + 'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'history_creator_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'history_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'history_id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'history_modifier_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'history_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'history_user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True'}), + 'id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'blank': 'True'}), + 'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '300'}), + 'organization_type_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}), + 'town': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.ishtaruser': { + 'Meta': {'object_name': 'IshtarUser', '_ormbases': ['auth.User']}, + 'person': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'ishtaruser'", 'unique': 'True', 'to': "orm['ishtar_common.Person']"}), + 'user_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'ishtar_common.organization': { + 'Meta': {'object_name': 'Organization'}, + 'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'history_creator': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'to': "orm['auth.User']"}), + 'history_modifier': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['auth.User']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '300'}), + 'organization_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.OrganizationType']"}), + 'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}), + 'town': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.organizationtype': { + 'Meta': {'ordering': "('label',)", 'object_name': 'OrganizationType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'ishtar_common.person': { + 'Meta': {'object_name': 'Person'}, + 'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'attached_to': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'members'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['ishtar_common.Organization']"}), + 'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'history_creator': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'to': "orm['auth.User']"}), + 'history_modifier': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['auth.User']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'person_types': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['ishtar_common.PersonType']", 'symmetrical': 'False'}), + 'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}), + 'raw_name': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}), + 'surname': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '2'}), + 'town': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.persontype': { + 'Meta': {'ordering': "('label',)", 'object_name': 'PersonType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': "orm['auth.Group']", 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'ishtar_common.sourcetype': { + 'Meta': {'object_name': 'SourceType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'ishtar_common.town': { + 'Meta': {'ordering': "['numero_insee']", 'object_name': 'Town'}, + 'canton': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Canton']", 'null': 'True', 'blank': 'True'}), + 'center': ('django.contrib.gis.db.models.fields.PointField', [], {'srid': '27572', 'null': 'True', 'blank': 'True'}), + 'departement': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Department']", 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'numero_insee': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '6'}), + 'surface': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}) + } + } + + complete_apps = ['ishtar_common']
\ No newline at end of file diff --git a/ishtar_common/models.py b/ishtar_common/models.py index f77c4f980..a96b24840 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -875,7 +875,10 @@ class Person(Address, OwnPerms, ValueGetter) : title = models.CharField(_(u"Title"), max_length=2, choices=TYPE) surname = models.CharField(_(u"Surname"), max_length=50, blank=True, null=True) - name = models.CharField(_(u"Name"), max_length=200) + name = models.CharField(_(u"Name"), max_length=200, blank=True, + null=True) + raw_name = models.CharField(_(u"Raw name"), max_length=300, blank=True, + null=True) person_types = models.ManyToManyField(PersonType, verbose_name=_(u"Types")) attached_to = models.ForeignKey('Organization', related_name='members', on_delete=models.SET_NULL, @@ -896,6 +899,8 @@ class Person(Address, OwnPerms, ValueGetter) : values = [unicode(getattr(self, attr)) for attr in ('surname', 'name') if getattr(self, attr)] + if not values: + values = [self.raw_name] if self.attached_to: values.append(u"- " + unicode(self.attached_to)) return u" ".join(values) |