diff options
| -rw-r--r-- | ishtar_common/data_importer.py | 74 | ||||
| -rw-r--r-- | ishtar_common/migrations/0026_auto__add_targetkey__add_unique_targetkey_target_value__add_field_impo.py | 340 | ||||
| -rw-r--r-- | ishtar_common/models.py | 106 | ||||
| -rw-r--r-- | ishtar_common/tests.py | 1 |
4 files changed, 489 insertions, 32 deletions
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index c2924a541..194a9a5fa 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -88,6 +88,9 @@ class ImporterError(Exception): return self.msg class Formater(object): + def __init__(self, *args, **kwargs): + self.db_target = kwargs.get('db_target', None) + def format(self, value): return value @@ -95,8 +98,10 @@ class Formater(object): return class UnicodeFormater(Formater): - def __init__(self, max_length, clean=False, re_filter=None, notnull=False): + def __init__(self, max_length, clean=False, re_filter=None, notnull=False, + db_target=None): self.max_length = max_length + self.db_target = db_target self.clean = clean self.re_filter = re_filter self.notnull = notnull @@ -182,12 +187,13 @@ class IntegerFormater(Formater): class StrChoiceFormater(Formater): def __init__(self, choices, strict=False, equiv_dict={}, model=None, - cli=False, many_split=''): + cli=False, many_split='', db_target=None): self.choices = list(choices) self.strict = strict self.equiv_dict = copy.deepcopy(equiv_dict) self.cli = cli self.model = model + self.db_target = db_target self.create = False self.missings = set() self.many_split = many_split @@ -200,6 +206,17 @@ class StrChoiceFormater(Formater): if model and v: v = model.objects.get(pk=v) self.equiv_dict[value] = v + if self.db_target: + for target_key in self.db_target.keys.filter(is_set=True).all(): + value = target_key.value + if not self.strict: + value = slugify(value) + if value in self.equiv_dict: + continue + v = target_key.key + if model and v: + v = model.objects.get(pk=v) + self.equiv_dict[value] = v def prepare(self, value): return unicode(value).strip() @@ -257,6 +274,13 @@ class StrChoiceFormater(Formater): unicode(self.equiv_dict[value]))) else: self.equiv_dict[value] = None + if output == 'db' and self.db_target: + for missing in missings: + try: + q = {'target':self.db_target, 'value':missing} + models.TargetKey.objects.create(**q) + except IntegrityError: + pass def new(self, value): return @@ -272,12 +296,14 @@ class StrChoiceFormater(Formater): return self.equiv_dict[value] class TypeFormater(StrChoiceFormater): - def __init__(self, model, cli=False, defaults={}, many_split=False): + def __init__(self, model, cli=False, defaults={}, many_split=False, + db_target=None): self.create = True self.strict = False self.model = model self.defaults = defaults self.many_split = many_split + self.db_target = db_target self.missings = set() self.equiv_dict, self.choices = {}, [] for item in model.objects.all(): @@ -304,8 +330,9 @@ class TypeFormater(StrChoiceFormater): return self.model.objects.create(**values) class DateFormater(Formater): - def __init__(self, date_format="%d/%m/%Y"): + def __init__(self, date_format="%d/%m/%Y", db_target=None): self.date_format = date_format + self.db_target = db_target def format(self, value): value = value.strip() @@ -318,11 +345,26 @@ class DateFormater(Formater): 'value':value}) class StrToBoolean(Formater): - def __init__(self, choices={}, cli=False, strict=False): + def __init__(self, choices={}, cli=False, strict=False, db_target=None): self.dct = copy.copy(choices) self.cli = cli self.strict= strict + self.db_target = db_target self.missings = set() + if self.db_target: + for target_key in self.db_target.keys.filter(is_set=True).all(): + value = target_key.value + value = self.prepare(value) + if value in self.dct: + continue + v = target_key.key + if v in ('False', '0'): + v = False + elif v: + v = True + else: + v = None + self.dct[value] = v def prepare(self, value): value = unicode(value).strip() @@ -359,6 +401,13 @@ class StrToBoolean(Formater): self.dct[value] = False else: self.dct[value] = None + if output == 'db' and self.db_target: + for missing in missings: + try: + q = {'target':self.db_target, 'value':missing} + models.TargetKey.objects.create(**q) + except IntegrityError: + pass def format(self, value): value = self.prepare(value) @@ -388,8 +437,9 @@ class Importer(object): } def __init__(self, skip_lines=0, reference_header=None, - check_col_num=False, test=False, check_validity=True, - history_modifier=None, output='silent'): + check_col_num=False, test=False, check_validity=True, + history_modifier=None, output='silent', + importer_instance=None): """ * skip_line must be set if the data provided has got headers lines. * a reference_header can be provided to perform a data compliance @@ -408,14 +458,18 @@ class Importer(object): self.check_col_num = check_col_num self.check_validity = check_validity self.line_format = copy.copy(self.LINE_FORMAT) + self.importer_instance = importer_instance self._initialized = False self._defaults = self.DEFAULTS.copy() self.history_modifier = history_modifier self.output = output if not self.history_modifier: - # get the first admin - self.history_modifier = User.objects.filter(is_superuser=True - ).order_by('pk')[0] + if self.importer_instance: + self.history_modifier = self.importer_instance.user + else: + # import made by the CLI: get the first admin + self.history_modifier = User.objects.filter( + is_superuser=True).order_by('pk')[0] def initialize(self, table, output='silent'): """ diff --git a/ishtar_common/migrations/0026_auto__add_targetkey__add_unique_targetkey_target_value__add_field_impo.py b/ishtar_common/migrations/0026_auto__add_targetkey__add_unique_targetkey_target_value__add_field_impo.py new file mode 100644 index 000000000..b4752a48e --- /dev/null +++ b/ishtar_common/migrations/0026_auto__add_targetkey__add_unique_targetkey_target_value__add_field_impo.py @@ -0,0 +1,340 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'TargetKey' + db.create_table('ishtar_common_targetkey', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('target', self.gf('django.db.models.fields.related.ForeignKey')(related_name='keys', to=orm['ishtar_common.ImporterColumn'])), + ('key', self.gf('django.db.models.fields.TextField')(null=True, blank=True)), + ('value', self.gf('django.db.models.fields.TextField')()), + ('is_set', self.gf('django.db.models.fields.BooleanField')(default=False)), + )) + db.send_create_signal('ishtar_common', ['TargetKey']) + + # Adding unique constraint on 'TargetKey', fields ['target', 'value'] + db.create_unique('ishtar_common_targetkey', ['target_id', 'value']) + + # Adding field 'Import.skip_lines' + db.add_column('ishtar_common_import', 'skip_lines', + self.gf('django.db.models.fields.IntegerField')(default=1), + keep_default=False) + + + # Changing field 'Import.creation_date' + db.alter_column('ishtar_common_import', 'creation_date', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, null=True)) + + def backwards(self, orm): + # Removing unique constraint on 'TargetKey', fields ['target', 'value'] + db.delete_unique('ishtar_common_targetkey', ['target_id', 'value']) + + # Deleting model 'TargetKey' + db.delete_table('ishtar_common_targetkey') + + # Deleting field 'Import.skip_lines' + db.delete_column('ishtar_common_import', 'skip_lines') + + + # Changing field 'Import.creation_date' + db.alter_column('ishtar_common_import', 'creation_date', self.gf('django.db.models.fields.DateTimeField')(null=True)) + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'ishtar_common.arrondissement': { + 'Meta': {'object_name': 'Arrondissement'}, + 'department': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Department']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '30'}) + }, + 'ishtar_common.author': { + 'Meta': {'object_name': 'Author'}, + 'author_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.AuthorType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'person': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'author'", 'to': "orm['ishtar_common.Person']"}) + }, + 'ishtar_common.authortype': { + 'Meta': {'object_name': 'AuthorType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'ishtar_common.canton': { + 'Meta': {'object_name': 'Canton'}, + 'arrondissement': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Arrondissement']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '30'}) + }, + 'ishtar_common.department': { + 'Meta': {'ordering': "['number']", 'object_name': 'Department'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '30'}), + 'number': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '3'}) + }, + 'ishtar_common.documenttemplate': { + 'Meta': {'ordering': "['associated_object_name', 'name']", 'object_name': 'DocumentTemplate'}, + 'associated_object_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'template': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}) + }, + 'ishtar_common.format': { + 'Meta': {'object_name': 'Format'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'ishtar_common.formatertype': { + 'Meta': {'unique_together': "(('formater_type', 'options', 'many_split'),)", 'object_name': 'FormaterType'}, + 'formater_type': ('django.db.models.fields.CharField', [], {'max_length': '20'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'many_split': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}), + 'options': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.globalvar': { + 'Meta': {'ordering': "['slug']", 'object_name': 'GlobalVar'}, + 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '50'}), + 'value': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.historicalorganization': { + 'Meta': {'ordering': "('-history_date', '-history_id')", 'object_name': 'HistoricalOrganization'}, + 'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'history_creator_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'history_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'history_id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'history_modifier_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'history_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'history_user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True'}), + 'id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'blank': 'True'}), + 'merge_key': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}), + 'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '300'}), + 'organization_type_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}), + 'town': ('django.db.models.fields.CharField', [], {'max_length': '70', 'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.import': { + 'Meta': {'object_name': 'Import'}, + 'creation_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'null': 'True', 'blank': 'True'}), + 'end_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'error_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'imported_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}), + 'importer_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.ImporterType']"}), + 'result_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'seconds_remaining': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'skip_lines': ('django.db.models.fields.IntegerField', [], {'default': '1'}), + 'state': ('django.db.models.fields.CharField', [], {'default': "'C'", 'max_length': '2'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.IshtarUser']"}) + }, + 'ishtar_common.importercolumn': { + 'Meta': {'object_name': 'ImporterColumn'}, + 'col_number': ('django.db.models.fields.IntegerField', [], {'default': '1'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'importer_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'columns'", 'to': "orm['ishtar_common.ImporterType']"}), + 'regexp_pre_filter': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Regexp']", 'null': 'True', 'blank': 'True'}), + 'required': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + }, + 'ishtar_common.importerdefault': { + 'Meta': {'object_name': 'ImporterDefault'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'importer_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'defaults'", 'to': "orm['ishtar_common.ImporterType']"}), + 'target': ('django.db.models.fields.CharField', [], {'max_length': '500'}) + }, + 'ishtar_common.importerdefaultvalues': { + 'Meta': {'object_name': 'ImporterDefaultValues'}, + 'default_target': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'default_values'", 'to': "orm['ishtar_common.ImporterDefault']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'target': ('django.db.models.fields.CharField', [], {'max_length': '500'}), + 'value': ('django.db.models.fields.CharField', [], {'max_length': '500'}) + }, + 'ishtar_common.importerduplicatefield': { + 'Meta': {'object_name': 'ImporterDuplicateField'}, + 'column': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'duplicate_fields'", 'to': "orm['ishtar_common.ImporterColumn']"}), + 'field_name': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + 'ishtar_common.importertype': { + 'Meta': {'object_name': 'ImporterType'}, + 'associated_models': ('django.db.models.fields.CharField', [], {'max_length': '200'}), + 'description': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_template': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'users': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': "orm['ishtar_common.IshtarUser']", 'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.importtarget': { + 'Meta': {'object_name': 'ImportTarget'}, + 'column': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'targets'", 'to': "orm['ishtar_common.ImporterColumn']"}), + 'formater_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.FormaterType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'regexp_filter': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Regexp']", 'null': 'True', 'blank': 'True'}), + 'target': ('django.db.models.fields.CharField', [], {'max_length': '500'}) + }, + 'ishtar_common.ishtaruser': { + 'Meta': {'object_name': 'IshtarUser', '_ormbases': ['auth.User']}, + 'person': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'ishtaruser'", 'unique': 'True', 'to': "orm['ishtar_common.Person']"}), + 'user_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True', 'primary_key': 'True'}) + }, + 'ishtar_common.itemkey': { + 'Meta': {'object_name': 'ItemKey'}, + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'importer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Import']", 'null': 'True', 'blank': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}) + }, + 'ishtar_common.organization': { + 'Meta': {'object_name': 'Organization'}, + 'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'history_creator': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['auth.User']"}), + 'history_modifier': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['auth.User']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'merge_candidate': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'merge_candidate_rel_+'", 'null': 'True', 'to': "orm['ishtar_common.Organization']"}), + 'merge_exclusion': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'merge_exclusion_rel_+'", 'null': 'True', 'to': "orm['ishtar_common.Organization']"}), + 'merge_key': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}), + 'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '300'}), + 'organization_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.OrganizationType']"}), + 'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}), + 'town': ('django.db.models.fields.CharField', [], {'max_length': '70', 'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.organizationtype': { + 'Meta': {'ordering': "('label',)", 'object_name': 'OrganizationType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'ishtar_common.person': { + 'Meta': {'object_name': 'Person'}, + 'address': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'address_complement': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'attached_to': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'members'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['ishtar_common.Organization']"}), + 'country': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'history_creator': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['auth.User']"}), + 'history_modifier': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['auth.User']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'merge_candidate': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'merge_candidate_rel_+'", 'null': 'True', 'to': "orm['ishtar_common.Person']"}), + 'merge_exclusion': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'merge_exclusion_rel_+'", 'null': 'True', 'to': "orm['ishtar_common.Person']"}), + 'merge_key': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}), + 'mobile_phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'person_types': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['ishtar_common.PersonType']", 'symmetrical': 'False'}), + 'phone': ('django.db.models.fields.CharField', [], {'max_length': '18', 'null': 'True', 'blank': 'True'}), + 'postal_code': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}), + 'raw_name': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}), + 'surname': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '2', 'null': 'True', 'blank': 'True'}), + 'town': ('django.db.models.fields.CharField', [], {'max_length': '70', 'null': 'True', 'blank': 'True'}) + }, + 'ishtar_common.persontype': { + 'Meta': {'ordering': "('label',)", 'object_name': 'PersonType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': "orm['auth.Group']", 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'ishtar_common.regexp': { + 'Meta': {'object_name': 'Regexp'}, + 'description': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'regexp': ('django.db.models.fields.CharField', [], {'max_length': '500'}) + }, + 'ishtar_common.sourcetype': { + 'Meta': {'object_name': 'SourceType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'ishtar_common.supporttype': { + 'Meta': {'object_name': 'SupportType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'txt_idx': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'ishtar_common.targetkey': { + 'Meta': {'unique_together': "(('target', 'value'),)", 'object_name': 'TargetKey'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_set': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'key': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'target': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'keys'", 'to': "orm['ishtar_common.ImporterColumn']"}), + 'value': ('django.db.models.fields.TextField', [], {}) + }, + 'ishtar_common.town': { + 'Meta': {'ordering': "['numero_insee']", 'object_name': 'Town'}, + 'canton': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Canton']", 'null': 'True', 'blank': 'True'}), + 'center': ('django.contrib.gis.db.models.fields.PointField', [], {'srid': '27572', 'null': 'True', 'blank': 'True'}), + 'departement': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['ishtar_common.Department']", 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'numero_insee': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '6'}), + 'surface': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}) + } + } + + complete_apps = ['ishtar_common']
\ No newline at end of file diff --git a/ishtar_common/models.py b/ishtar_common/models.py index f2193eae2..1f059f2f5 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -28,6 +28,7 @@ from importlib import import_module import os import re import tempfile +import unicodecsv from django.conf import settings from django.core.cache import cache @@ -955,13 +956,6 @@ class OrganizationType(GeneralType): verbose_name_plural = _(u"Organization types") ordering = ('label',) -IMPORT_STATE = (("C", _(u"Created")), - ("AP", _(u"Analyse in progress")), - ("A", _(u"Analysed")), - ("P", _(u"Import pending")), - ("IP", _(u"Import in progress")), - ("F", _(u"Finished"))) - MODELS = [ ('archaeological_operations.models.Operation', _(u"Operation")), ('archaeological_operations.models.Parcel', _(u"Parcels")), @@ -985,6 +979,9 @@ def get_model_fields(model): return fields class ImporterType(models.Model): + """ + Description of a table to be mapped with ishtar database + """ name = models.CharField(_(u"Name"), blank=True, null=True, max_length=100) description = models.CharField(_(u"Description"), blank=True, null=True, @@ -1022,7 +1019,7 @@ class ImporterType(models.Model): LINE_FORMAT.append(None) continue for target in column.targets.all(): - ft = target.formater_type.get_formater_type() + ft = target.formater_type.get_formater_type(target) if not ft: continue formater_types.append(ft) @@ -1032,8 +1029,9 @@ class ImporterType(models.Model): formater_kwargs['regexp'] = re.compile( column.regexp_pre_filter.regexp) formater_kwargs['duplicate_fields'] = [field.field_name - for field in column.duplicate_fields.all()] - formater = ImportFormater(targets, formater_types, **formater_kwargs) + for field in column.duplicate_fields.all()] + formater = ImportFormater(targets, formater_types, + **formater_kwargs) LINE_FORMAT.append(formater) args = {'OBJECT_CLS':OBJECT_CLS, 'DESC':self.description, 'DEFAULTS':DEFAULTS, 'LINE_FORMAT':LINE_FORMAT} @@ -1041,6 +1039,9 @@ class ImporterType(models.Model): return newclass class ImporterDefault(models.Model): + """ + Targets of default values in an import + """ importer_type = models.ForeignKey(ImporterType, related_name='defaults') target = models.CharField(u"Target", max_length=500) class Meta: @@ -1071,6 +1072,9 @@ class ImporterDefault(models.Model): return values class ImporterDefaultValues(models.Model): + """ + Default values in an import + """ default_target = models.ForeignKey(ImporterDefault, related_name='default_values') target = models.CharField(u"Target", max_length=500) @@ -1096,6 +1100,9 @@ class ImporterDefaultValues(models.Model): return "" class ImporterColumn(models.Model): + """ + Import file column description + """ importer_type = models.ForeignKey(ImporterType, related_name='columns') col_number = models.IntegerField(_(u"Column number"), default=1) regexp_pre_filter = models.ForeignKey("Regexp", blank=True, null=True) @@ -1105,6 +1112,9 @@ class ImporterColumn(models.Model): verbose_name_plural = _(u"Importer - Columns") class ImporterDuplicateField(models.Model): + """ + Direct copy of result in other fields + """ column = models.ForeignKey(ImporterColumn, related_name='duplicate_fields') field_name = models.CharField(_(u"Field name"), blank=True, null=True, max_length=200) @@ -1124,6 +1134,9 @@ class Regexp(models.Model): IMPORTER_TYPES = [] class ImportTarget(models.Model): + """ + Ishtar database target for a column + """ column = models.ForeignKey(ImporterColumn, related_name='targets') target = models.CharField(u"Target", max_length=500) regexp_filter = models.ForeignKey("Regexp", blank=True, null=True) @@ -1132,6 +1145,26 @@ class ImportTarget(models.Model): verbose_name = _(u"Importer - Target") verbose_name_plural = _(u"Importer - Targets") + def __unicode__(self): + return u" - ".join([unicode(self.column), self.target[:50]]) + +class TargetKey(models.Model): + """ + User's link between import source and ishtar database. + Also temporary used for GeneralType to point missing link before adding + them in ItemKey table + """ + target = models.ForeignKey(ImporterColumn, related_name='keys') + key = models.TextField(_(u"Key"), blank=True, null=True) + value = models.TextField(_(u"Value")) + is_set = models.BooleanField(_(u"Is set"), default=False) + + class Meta: + unique_together = ('target', 'value') + + def __unicode__(self): + return u" - ".join([unicode(self.target), self.key[:50]]) + TARGET_MODELS = [ ('OrganizationType', _(u"Organization type")), ('SourceType', _(u"Source type")), @@ -1190,10 +1223,10 @@ class FormaterType(models.Model): if self.format_type in IMPORTER_TYPES_CHOICES: return IMPORTER_TYPES_CHOICES[self.format_type] - def get_formater_type(self): + def get_formater_type(self, target): if self.formater_type not in IMPORTER_TYPES_DCT.keys(): return - kwargs = {} + kwargs = {'target':target} if self.many_split: kwargs['many_split'] = self.many_split if self.formater_type == 'TypeFormater': @@ -1206,35 +1239,44 @@ class FormaterType(models.Model): model = import_module(self.options) return TypeFormater(model, **kwargs) elif self.formater_type == 'IntegerFormater': - return IntegerFormater() + return IntegerFormater(**kwargs) elif self.formater_type == 'FloatFormater': - return FloatFormater() + return FloatFormater(**kwargs) elif self.format_type == 'UnicodeFormater': try: - return UnicodeFormater(int(self.options.strip())) + return UnicodeFormater(int(self.options.strip()), **kwargs) except ValueError: return elif self.format_type == 'DateFormater': - return DateFormater(self.options) + return DateFormater(self.options, **kwargs) + +IMPORT_STATE = (("C", _(u"Created")), + ("AP", _(u"Analyse in progress")), + ("A", _(u"Analysed")), + ("P", _(u"Import pending")), + ("IP", _(u"Import in progress")), + ("F", _(u"Finished"))) class Import(models.Model): user = models.ForeignKey('IshtarUser') importer_type = models.ForeignKey(ImporterType) imported_file = models.FileField(_(u"Imported file"), upload_to="upload/imports/") + skip_lines = models.IntegerField(default=1) error_file = models.FileField(_(u"Error file"), upload_to="upload/imports/", blank=True, null=True) result_file = models.FileField(_(u"Result file"), upload_to="upload/imports/", blank=True, null=True) - state = models.CharField(_(u"State"), max_length=2, choices=IMPORT_STATE) - creation_date = models.DateTimeField(_(u"Creation date"), blank=True, - null=True) + state = models.CharField(_(u"State"), max_length=2, choices=IMPORT_STATE, + default='C') + creation_date = models.DateTimeField(_(u"Creation date"), auto_now_add=True, + blank=True, null=True) end_date = models.DateTimeField(_(u"End date"), blank=True, - null=True) + null=True, editable=False) seconds_remaining = models.IntegerField(_(u"Seconds remaining"), blank=True, - null=True) + null=True, editable=False) class Meta: verbose_name = _(u"Import") verbose_name_plural = _(u"Imports") @@ -1243,6 +1285,28 @@ class Import(models.Model): return u"%s - %s" % (unicode(self.importer_type), unicode(self.user)) + @property + def importer_instance(self): + return self.importer_type.importer_class(skip_lines=self.skip_lines, + import_instance=self) + + @property + def data_table(self): + encodings = [settings.ENCODING, settings.ALT_ENCODING, 'utf-8'] + with open(self.imported_file.filename) as csv_file: + for encoding in encodings: + try: + return [line for line in unicodecsv.reader(csv_file, + encoding=encoding)] + except UnicodeDecodeError: + if encoding != encodings[-1]: + csv_file.seek(0) + continue + return [] + + def initialize(self): + self.importer_instance.initialize(self.data_table, output='db') + class Organization(Address, Merge, OwnPerms, ValueGetter): TABLE_COLS = ('name', 'organization_type',) name = models.CharField(_(u"Name"), max_length=300) diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py index 4b4f38e4a..8745b079d 100644 --- a/ishtar_common/tests.py +++ b/ishtar_common/tests.py @@ -103,7 +103,6 @@ class MergeTest(TestCase): self.assertTrue(self.person_types[1] in self.person_3.person_types.all()) class ImportKeyTest(TestCase): - def testKeys(self): content_type = ContentType.objects.get_for_model(models.OrganizationType) |