diff options
Diffstat (limited to 'ishtar_common')
| -rw-r--r-- | ishtar_common/data_importer.py | 318 | ||||
| -rw-r--r-- | ishtar_common/fixtures/initial_importtypes-fr.json | 4 | ||||
| -rw-r--r-- | ishtar_common/forms_common.py | 12 | ||||
| -rw-r--r-- | ishtar_common/models.py | 65 | ||||
| -rw-r--r-- | ishtar_common/tests.py | 11 | ||||
| -rw-r--r-- | ishtar_common/wizards.py | 6 |
6 files changed, 245 insertions, 171 deletions
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index a03f4de34..426d32a7a 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -29,9 +29,10 @@ import zipfile from django.conf import settings from django.contrib.auth.models import User -from django.core.exceptions import ImproperlyConfigured +from django.db.models.fields import FieldDoesNotExist from django.core.files import File from django.db import IntegrityError, DatabaseError, transaction +from django.db.models import Q from django.template.defaultfilters import slugify from django.utils.translation import ugettext_lazy as _ @@ -148,13 +149,15 @@ class ChoiceChecker(object): class UnicodeFormater(Formater): def __init__(self, max_length=None, clean=False, re_filter=None, - notnull=False, prefix=u'', db_target=None): + notnull=False, prefix=u'', db_target=None, + import_instance=None): self.max_length = max_length self.db_target = db_target self.clean = clean self.re_filter = re_filter self.notnull = notnull self.prefix = prefix + self.import_instance = import_instance def format(self, value): try: @@ -255,7 +258,8 @@ class IntegerFormater(Formater): class StrChoiceFormater(Formater, ChoiceChecker): def __init__(self, choices, strict=False, equiv_dict={}, model=None, - cli=False, many_split='', db_target=None): + cli=False, many_split='', db_target=None, + import_instance=None): self.choices = list(choices) self.strict = strict self.equiv_dict = copy.deepcopy(equiv_dict) @@ -267,6 +271,7 @@ class StrChoiceFormater(Formater, ChoiceChecker): self.new_keys = {} self.match_table = {} self.many_split = many_split + self.import_instance = None for key, value in self.choices: value = unicode(value) if not self.strict: @@ -281,7 +286,11 @@ class StrChoiceFormater(Formater, ChoiceChecker): def init_db_target(self): if not self.db_target: return - for target_key in self.db_target.keys.filter(is_set=True).all(): + q = self.db_target.keys.filter(is_set=True) + if self.import_instance: + q = q.filter(Q(associated_import=self.import_instance) | + Q(associated_import__isnull=True)) + for target_key in q.all(): key = target_key.key if not self.strict: key = slugify(key) @@ -429,7 +438,7 @@ class StrChoiceFormater(Formater, ChoiceChecker): class TypeFormater(StrChoiceFormater): def __init__(self, model, cli=False, defaults={}, many_split=False, - db_target=None): + db_target=None, import_instance=None): self.create = True self.strict = False self.model = model @@ -440,9 +449,10 @@ class TypeFormater(StrChoiceFormater): self.equiv_dict, self.choices = {}, [] self.match_table = {} self.new_keys = {} + self.import_instance = import_instance for item in model.objects.all(): self.choices.append((item.pk, unicode(item))) - for key in item.get_keys(): + for key in item.get_keys(importer_id=import_instance.pk): self.equiv_dict[key] = item def prepare(self, value): @@ -465,11 +475,13 @@ class TypeFormater(StrChoiceFormater): class DateFormater(Formater): - def __init__(self, date_formats=["%d/%m/%Y"], db_target=None): + def __init__(self, date_formats=["%d/%m/%Y"], db_target=None, + import_instance=None): self.date_formats = date_formats if type(date_formats) not in (list, tuple): self.date_formats = [self.date_formats] self.db_target = db_target + self.import_instance = import_instance def format(self, value): value = value.strip() @@ -511,7 +523,8 @@ class FileFormater(Formater): class StrToBoolean(Formater, ChoiceChecker): - def __init__(self, choices={}, cli=False, strict=False, db_target=None): + def __init__(self, choices={}, cli=False, strict=False, db_target=None, + import_instance=None): self.dct = copy.copy(choices) self.cli = cli self.strict = strict @@ -520,6 +533,7 @@ class StrToBoolean(Formater, ChoiceChecker): self.init_db_target() self.match_table = {} self.new_keys = {} + self.import_instance = import_instance def init_db_target(self): if not self.db_target: @@ -733,7 +747,7 @@ class Importer(object): q = ImporterModel.objects.filter(klass=cls_name) if q.count(): cls_name = q.all()[0].name - return ImproperlyConfigured( + return ImporterError( unicode(self.ERRORS['improperly_configured']).format(cls_name)) def _get_does_not_exist_in_db_error(self, model, data): @@ -827,7 +841,7 @@ class Importer(object): vals[idx_col].append(val) for idx, formater in enumerate(self.line_format): if formater and idx < len(vals): - + formater.import_instance = self.import_instance if self.DB_TARGETS: field_names = formater.field_name if type(field_names) not in (list, tuple): @@ -1143,6 +1157,7 @@ class Importer(object): self.concat_str[field_name] = concat_str if self.DB_TARGETS: + formater.import_instance = self.import_instance formater.reinit_db_target( self.DB_TARGETS["{}-{}".format(idx_col + 1, field_name)], idx_v) @@ -1217,8 +1232,13 @@ class Importer(object): c_row.append(u" ; ".join([v for v in c_values])) def get_field(self, cls, attribute, data, m2ms, c_path, new_created): - field_object, model, direct, m2m = \ - cls._meta.get_field_by_name(attribute) + try: + field_object, model, direct, m2m = \ + cls._meta.get_field_by_name(attribute) + except FieldDoesNotExist: + raise ImporterError(unicode( + _(u"Importer configuration error: field \"{}\" does not exist " + u"for {}.")).format(attribute, cls._meta.verbose_name)) if m2m: many_values = data.pop(attribute) if hasattr(field_object, 'rel'): @@ -1361,146 +1381,152 @@ class Importer(object): def get_object(self, cls, data, path=[]): m2ms = [] - if data and type(data) == dict: - c_path = path[:] - - # get all related fields - new_created = {} - for attribute in list(data.keys()): - c_c_path = c_path[:] - if not attribute: - data.pop(attribute) - continue - if not data[attribute]: - continue - if attribute != '__force_new': - self.get_field(cls, attribute, data, m2ms, c_c_path, - new_created) - - create_dict = copy.deepcopy(data) - for k in create_dict.keys(): - # filter unnecessary default values - if type(create_dict[k]) == dict: - create_dict.pop(k) - # File doesn't like deepcopy - if type(create_dict[k]) == File: - create_dict[k] = copy.copy(data[k]) - - # default values - path = tuple(path) - defaults = {} - if path in self._defaults: - for k in self._defaults[path]: - if (k not in data or not data[k]): - defaults[k] = self._defaults[path][k] - - if 'history_modifier' in create_dict: - defaults.update({ - 'history_modifier': create_dict.pop('history_modifier') - }) - - created = False + if type(data) != dict: + return data, False + is_empty = not bool( + [k for k in data if k not in ('history_modifier', 'defaults') + and data[k]]) + if is_empty: + return None, False + + c_path = path[:] + + # get all related fields + new_created = {} + for attribute in list(data.keys()): + c_c_path = c_path[:] + if not attribute: + data.pop(attribute) + continue + if not data[attribute]: + continue + if attribute != '__force_new': + self.get_field(cls, attribute, data, m2ms, c_c_path, + new_created) + + create_dict = copy.deepcopy(data) + for k in create_dict.keys(): + # filter unnecessary default values + if type(create_dict[k]) == dict: + create_dict.pop(k) + # File doesn't like deepcopy + if type(create_dict[k]) == File: + create_dict[k] = copy.copy(data[k]) + + # default values + path = tuple(path) + defaults = {} + if path in self._defaults: + for k in self._defaults[path]: + if k not in data or not data[k]: + defaults[k] = self._defaults[path][k] + + if 'history_modifier' in create_dict: + defaults.update({ + 'history_modifier': create_dict.pop('history_modifier') + }) + + created = False + try: try: - try: - dct = create_dict.copy() - for key in dct: - if callable(dct[key]): - dct[key] = dct[key]() - if '__force_new' in dct: - created = dct.pop('__force_new') - if not [k for k in dct if dct[k] is not None]: - return None, created - new_dct = defaults.copy() - new_dct.update(dct) - if self.MODEL_CREATION_LIMIT and \ - cls not in self.MODEL_CREATION_LIMIT: - raise self._get_improperly_conf_error(cls) - obj = cls.objects.create(**new_dct) + dct = create_dict.copy() + for key in dct: + if callable(dct[key]): + dct[key] = dct[key]() + if '__force_new' in dct: + created = dct.pop('__force_new') + if not [k for k in dct if dct[k] is not None]: + return None, created + new_dct = defaults.copy() + new_dct.update(dct) + if self.MODEL_CREATION_LIMIT and \ + cls not in self.MODEL_CREATION_LIMIT: + raise self._get_improperly_conf_error(cls) + obj = cls.objects.create(**new_dct) + else: + # manage UNICITY_KEYS - only level 1 + if not path and self.UNICITY_KEYS: + for k in dct.keys(): + if k not in self.UNICITY_KEYS \ + and k != 'defaults': + defaults[k] = dct.pop(k) + if not self.MODEL_CREATION_LIMIT or \ + cls in self.MODEL_CREATION_LIMIT: + dct['defaults'] = defaults.copy() + obj, created = cls.objects.get_or_create(**dct) else: - # manage UNICITY_KEYS - only level 1 - if not path and self.UNICITY_KEYS: - for k in dct.keys(): - if k not in self.UNICITY_KEYS \ - and k != 'defaults': - defaults[k] = dct.pop(k) - if not self.MODEL_CREATION_LIMIT or \ - cls in self.MODEL_CREATION_LIMIT: + try: + obj = cls.objects.get(**dct) dct['defaults'] = defaults.copy() - obj, created = cls.objects.get_or_create(**dct) - else: - try: - obj = cls.objects.get(**dct) - dct['defaults'] = defaults.copy() - except cls.DoesNotExist: - raise self._get_does_not_exist_in_db_error( - cls, dct) - - if not created and not path and self.UNICITY_KEYS: - changed = False - if self.conservative_import: - for k in dct['defaults']: - new_val = dct['defaults'][k] - if new_val is None or new_val == '': - continue - val = getattr(obj, k) - if val is None or val == '': - changed = True - setattr(obj, k, new_val) - elif k in self.concats \ - and type(val) == unicode \ - and type(new_val) == unicode: - setattr(obj, k, val + u"\n" + new_val) - else: - for k in dct['defaults']: - new_val = dct['defaults'][k] - if new_val is None or new_val == '': - continue + except cls.DoesNotExist: + raise self._get_does_not_exist_in_db_error( + cls, dct) + + if not created and not path and self.UNICITY_KEYS: + changed = False + if self.conservative_import: + for k in dct['defaults']: + new_val = dct['defaults'][k] + if new_val is None or new_val == '': + continue + val = getattr(obj, k) + if val is None or val == '': changed = True setattr(obj, k, new_val) - if changed: - obj.save() - if self.import_instance and hasattr(obj, 'imports') \ - and created: - obj.imports.add(self.import_instance) - except ValueError as e: - raise IntegrityError(e.message) - except IntegrityError as e: - raise IntegrityError(e.message) - except DatabaseError as e: - raise IntegrityError(e.message) - except cls.MultipleObjectsReturned as e: - created = False - if 'defaults' in dct: - dct.pop('defaults') - raise IntegrityError(e.message) - # obj = cls.objects.filter(**dct).all()[0] - for attr, value in m2ms: - values = [value] - if type(value) in (list, tuple): - values = value - for v in values: - getattr(obj, attr).add(v) - # force post save script - v.save() - if m2ms: - # force post save script - obj.save() + elif k in self.concats \ + and type(val) == unicode \ + and type(new_val) == unicode: + setattr(obj, k, val + u"\n" + new_val) + else: + for k in dct['defaults']: + new_val = dct['defaults'][k] + if new_val is None or new_val == '': + continue + changed = True + setattr(obj, k, new_val) + if changed: + obj.save() + if self.import_instance and hasattr(obj, 'imports') \ + and created: + obj.imports.add(self.import_instance) + except ValueError as e: + raise IntegrityError(e.message) except IntegrityError as e: - message = e.message - try: - message = e.message.decode('utf-8') - except (UnicodeDecodeError, UnicodeDecodeError): - message = '' - try: - data = unicode(data) - except UnicodeDecodeError: - data = '' - raise ImporterError( - "Erreur d'import %s %s, contexte : %s, erreur : %s" - % (unicode(cls), unicode("__".join(path)), - unicode(data), message)) - return obj, created - return data + raise IntegrityError(e.message) + except DatabaseError as e: + raise IntegrityError(e.message) + except cls.MultipleObjectsReturned as e: + created = False + if 'defaults' in dct: + dct.pop('defaults') + raise IntegrityError(e.message) + # obj = cls.objects.filter(**dct).all()[0] + for attr, value in m2ms: + values = [value] + if type(value) in (list, tuple): + values = value + for v in values: + getattr(obj, attr).add(v) + # force post save script + v.save() + if m2ms: + # force post save script + obj.save() + except IntegrityError as e: + message = e.message + try: + message = e.message.decode('utf-8') + except (UnicodeDecodeError, UnicodeDecodeError): + message = '' + try: + data = unicode(data) + except UnicodeDecodeError: + data = '' + raise ImporterError( + "Erreur d'import %s %s, contexte : %s, erreur : %s" + % (unicode(cls), unicode("__".join(path)), + unicode(data), message)) + return obj, created def _format_csv_line(self, values, empty=u"-"): return u'"' + u'","'.join( diff --git a/ishtar_common/fixtures/initial_importtypes-fr.json b/ishtar_common/fixtures/initial_importtypes-fr.json index 1b155ddf6..9ee6710c4 100644 --- a/ishtar_common/fixtures/initial_importtypes-fr.json +++ b/ishtar_common/fixtures/initial_importtypes-fr.json @@ -100,7 +100,7 @@ "description": "", "created_models": [], "is_template": true, - "unicity_keys": "external_id", + "unicity_keys": "code_patriarche", "users": [], "slug": "mcc-operations", "associated_models": 6, @@ -5306,4 +5306,4 @@ "force_new": false } } -]
\ No newline at end of file +] diff --git a/ishtar_common/forms_common.py b/ishtar_common/forms_common.py index 7ab09f9f7..e2c0e5db5 100644 --- a/ishtar_common/forms_common.py +++ b/ishtar_common/forms_common.py @@ -135,7 +135,9 @@ class TargetKeyForm(forms.ModelForm): def __init__(self, *args, **kwargs): super(TargetKeyForm, self).__init__(*args, **kwargs) instance = getattr(self, 'instance', None) + self.associated_import = None if instance and instance.pk: + self.associated_import = instance.associated_import self.fields['target'].widget.attrs['readonly'] = True self.fields['key'].widget.attrs['readonly'] = True self.fields['key'].widget.attrs['title'] = unicode(instance) @@ -162,7 +164,7 @@ class TargetKeyForm(forms.ModelForm): super(TargetKeyForm, self).save(commit) if self.cleaned_data.get('value'): self.instance.is_set = True - self.associated_import = None + self.instance.associated_import = self.associated_import self.instance.save() @@ -579,6 +581,7 @@ class AccountForm(forms.Form): widget=forms.PasswordInput, required=False) def __init__(self, *args, **kwargs): + person = None if 'initial' in kwargs and 'pk' in kwargs['initial']: try: person = models.Person.objects.get(pk=kwargs['initial']['pk']) @@ -589,7 +592,12 @@ class AccountForm(forms.Form): kwargs['initial']['email'] = account.email except ObjectDoesNotExist: pass - return super(AccountForm, self).__init__(*args, **kwargs) + if 'person' in kwargs: + person = kwargs.pop('person') + super(AccountForm, self).__init__(*args, **kwargs) + if person: + self.fields['username'].initial = \ + person.raw_name.lower().replace(' ', '.') def clean(self): cleaned_data = self.cleaned_data diff --git a/ishtar_common/models.py b/ishtar_common/models.py index 90e2bd6f6..2496e4372 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -22,6 +22,7 @@ Models description """ from cStringIO import StringIO import copy +import csv import datetime from PIL import Image from importlib import import_module @@ -72,17 +73,18 @@ logger = logging.getLogger(__name__) def post_save_user(sender, **kwargs): user = kwargs['instance'] + try: q = IshtarUser.objects.filter(username=user.username) if not q.count(): ishtaruser = IshtarUser.create_from_user(user) else: ishtaruser = q.all()[0] - ADMINISTRATOR, created = PersonType.objects.get_or_create( + administrator, created = PersonType.objects.get_or_create( txt_idx='administrator') if ishtaruser.is_superuser \ and not ishtaruser.has_right('administrator'): - ishtaruser.person.person_types.add(ADMINISTRATOR) + ishtaruser.person.person_types.add(administrator) except DatabaseError: # manage when db is not synced pass post_save.connect(post_save_user, sender=User) @@ -648,27 +650,34 @@ class GeneralType(Cached, models.Model): self.generate_key(force=True) return obj - def add_key(self, key, force=False): + def add_key(self, key, force=False, importer=None): content_type = ContentType.objects.get_for_model(self.__class__) - if not force and ItemKey.objects.filter( + if not importer and not force and ItemKey.objects.filter( key=key, content_type=content_type).count(): return if force: - ItemKey.objects.filter(key=key, content_type=content_type)\ + ItemKey.objects.filter(key=key, content_type=content_type, + importer=importer)\ .exclude(object_id=self.pk).delete() - ItemKey.objects.get_or_create(object_id=self.pk, key=key, - content_type=content_type) + ItemKey.objects.get_or_create( + object_id=self.pk, key=key, content_type=content_type, + importer=importer + ) def generate_key(self, force=False): for key in (slugify(self.label), self.txt_idx): self.add_key(key) - def get_keys(self): + def get_keys(self, importer_id=None): keys = [self.txt_idx] content_type = ContentType.objects.get_for_model(self.__class__) - for ik in ItemKey.objects.filter( - content_type=content_type, object_id=self.pk).exclude( - key=self.txt_idx).all(): + query = Q(content_type=content_type, object_id=self.pk, + importer__isnull=True) + if importer_id: + query |= Q(content_type=content_type, object_id=self.pk, + importer__pk=importer_id) + q = ItemKey.objects.filter(query) + for ik in q.exclude(key=self.txt_idx).all(): keys.append(ik.key) return keys @@ -1782,7 +1791,7 @@ class ImporterType(models.Model): def __unicode__(self): return self.name - def get_importer_class(self): + def get_importer_class(self, import_instance=None): if self.slug and self.slug in IMPORTER_CLASSES: cls = import_class(IMPORTER_CLASSES[self.slug]) return cls @@ -1805,7 +1814,8 @@ class ImporterType(models.Model): force_news = [] concat_str = [] for target in column.targets.all(): - ft = target.formater_type.get_formater_type(target) + ft = target.formater_type.get_formater_type( + target, import_instance=import_instance) if not ft: continue formater_types.append(ft) @@ -2106,10 +2116,10 @@ class TargetKey(models.Model): try: v = self.target.associated_model.objects.get( txt_idx=unicode(self.value)) - except (self.target.associated_model.DoesNotExist): + except self.target.associated_model.DoesNotExist: pass if v: - v.add_key(self.key) + v.add_key(self.key, importer=self.associated_import) return obj TARGET_MODELS = [ @@ -2203,10 +2213,10 @@ class FormaterType(models.Model): if self.format_type in IMPORTER_TYPES_CHOICES: return IMPORTER_TYPES_CHOICES[self.format_type] - def get_formater_type(self, target): + def get_formater_type(self, target, import_instance=None): if self.formater_type not in IMPORTER_TYPES_DCT.keys(): return - kwargs = {'db_target': target} + kwargs = {'db_target': target, 'import_instance': import_instance} if self.many_split: kwargs['many_split'] = self.many_split if self.formater_type == 'TypeFormater': @@ -2302,6 +2312,19 @@ class Import(models.Model): return bool(TargetKey.objects.filter(associated_import=self, is_set=False).count()) + @property + def errors(self): + if not self.error_file: + return [] + errors = [] + with open(self.error_file.path, 'rb') as csvfile: + reader = csv.DictReader(csvfile, fieldnames=['line', 'column', + 'error']) + reader.next() # pass the header + for row in reader: + errors.append(row) + return errors + def get_actions(self): """ Get available action relevant with the current status @@ -2332,7 +2355,7 @@ class Import(models.Model): return IMPORT_STATE_DCT[self.state] def get_importer_instance(self): - return self.importer_type.get_importer_class()( + return self.importer_type.get_importer_class(import_instance=self)( skip_lines=self.skip_lines, import_instance=self, conservative_import=self.conservative_import) @@ -2783,8 +2806,10 @@ class IshtarUser(User): name=name, email=email, history_modifier=user) person.person_types.add(person_type) - return IshtarUser.objects.create(user_ptr=user, username=default, - person=person) + password = user.password + isht_user = IshtarUser.objects.create( + user_ptr=user, username=default, person=person, password=password) + return isht_user def has_right(self, right_name, session=None): return self.person.has_right(right_name, session=session) diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py index 82acb1904..42bb1860e 100644 --- a/ishtar_common/tests.py +++ b/ishtar_common/tests.py @@ -319,6 +319,15 @@ class AdminGenTypeTest(TestCase): for model in self.models_with_data: self.assertTrue(str(model.objects.all()[0])) + def test_user_creation(self): + url = '/admin/auth/user/add/' + password = 'ishtar is the queen' + response = self.client.post( + url, {'username': 'test', 'password1': password, + 'password2': password}) + self.assertEqual(response.status_code, 302) + self.assertTrue(self.client.login(username='test', password=password)) + class MergeTest(TestCase): def setUp(self): @@ -798,7 +807,7 @@ class ImportTest(TestCase): # town should be deleted self.assertEqual(models.Town.objects.filter(name='my-test').count(), 0) - def testKeys(self): + def test_keys(self): content_type = ContentType.objects.get_for_model( models.OrganizationType) diff --git a/ishtar_common/wizards.py b/ishtar_common/wizards.py index 5105418a5..c2cd54d03 100644 --- a/ishtar_common/wizards.py +++ b/ishtar_common/wizards.py @@ -1343,6 +1343,12 @@ class AccountWizard(Wizard): context_instance=RequestContext(self.request)) return res + def get_form_kwargs(self, step=None): + kwargs = super(AccountWizard, self).get_form_kwargs(step) + if step == 'account-account_management': + kwargs['person'] = self.get_current_object() + return kwargs + def get_form(self, step=None, data=None, files=None): """ Display the "Send email" field if necessary |