diff options
Diffstat (limited to 'ishtar_common')
| -rw-r--r-- | ishtar_common/data_importer.py | 18 | ||||
| -rw-r--r-- | ishtar_common/forms_common.py | 16 | ||||
| -rw-r--r-- | ishtar_common/models_imports.py | 102 | ||||
| -rw-r--r-- | ishtar_common/utils.py | 47 |
4 files changed, 138 insertions, 45 deletions
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index 6a58e0409..d5e6f966c 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -748,6 +748,7 @@ class Importer(object): MODEL_CREATION_LIMIT = [] EXTRA_DEFAULTS = {} DEFAULTS = {} + PRE_IMPORT_VALUES = {} # values from a form before the import ERRORS = { "header_check": _( "The given file is not correct. Check the file " @@ -839,6 +840,7 @@ class Importer(object): if k not in self._defaults: self._defaults[k] = {} self._defaults[k].update(self.EXTRA_DEFAULTS[k]) + self._pre_import_values = self.PRE_IMPORT_VALUES.copy() self.history_modifier = history_modifier self.output = output if not self.history_modifier: @@ -1151,7 +1153,7 @@ class Importer(object): idx_col = 0 for idx_col, val in enumerate(line): try: - self._row_processing(c_row, idx_col, idx_line, val, data) + data = self._row_processing(c_row, idx_col, idx_line, val, data) # nosec: no catch to force continue processing of lines except: # nosec pass @@ -1367,7 +1369,7 @@ class Importer(object): def _row_processing(self, c_row, idx_col, idx_line, val, data): if idx_col >= len(self.line_format): - return + return data formater = self.line_format[idx_col] @@ -1376,7 +1378,7 @@ class Importer(object): if not formater or not formater.field_name: c_row.append(_("Not imported")) - return + return data if formater.regexp: # multiline regexp is a mess... @@ -1390,7 +1392,7 @@ class Importer(object): self.c_errors = True elif not val.strip(): c_row.append("") - return + return data val = val.replace(NEW_LINE_BREAK, "\n") self.errors.append( ( @@ -1400,7 +1402,7 @@ class Importer(object): ) ) c_row.append("") - return + return data val_group = [] for g in formater.regexp.findall(val): if isinstance(g, (tuple, list)): @@ -1470,7 +1472,7 @@ class Importer(object): self.c_errors = True self.errors.append((idx_line + 1, idx_col + 1, str(e))) c_values.append("") - return + return data if formater.value_format and value is not None and value != "": if "{item" in formater.value_format: value = formater.value_format.format(item=value) @@ -1503,7 +1505,7 @@ class Importer(object): self.errors.append( (idx_line + 1, idx_col + 1, self.ERRORS["value_required"]) ) - return + return data field_names = [field_name] force_news = [force_new] @@ -1536,7 +1538,9 @@ class Importer(object): concat=concats[idx], concat_str=concat_str[idx], ) + data = update_data(self._pre_import_values, data) c_row.append(" ; ".join([v for v in c_values])) + return data def _get_field_m2m( self, attribute, data, c_path, new_created, field_object, idx_line=None diff --git a/ishtar_common/forms_common.py b/ishtar_common/forms_common.py index 3e3488ae9..cc61f4da7 100644 --- a/ishtar_common/forms_common.py +++ b/ishtar_common/forms_common.py @@ -671,19 +671,9 @@ class PreImportForm(IshtarForm): attrs["widget"] = getattr(forms, widget_name) key = f"col_{- column.col_number}" form_field = getattr(forms, field_name) - if field_name == "ChoiceField": - if not target.formater_type.options: - continue - options = target.formater_type.options.split(".") - if len(options) == 1: - app = "ishtar_common" - else: - app = options[0] - model_name = options[-1] - try: - model = apps.get_model(app, model_name) - except LookupError: - continue + + model = target.formater_type.associated_model + if model: if target.formater_type.many_split: form_field = widgets.Select2MultipleField self._types.append( diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py index 454bed7e8..c03dd208b 100644 --- a/ishtar_common/models_imports.py +++ b/ishtar_common/models_imports.py @@ -65,14 +65,16 @@ from ishtar_common.model_managers import SlugModelManager from ishtar_common.utils import ( create_slug, + generate_dict_from_list, get_all_related_m2m_objects_with_model, - put_session_message, - put_session_var, get_session_var, - num2col, - max_size_help, import_class, + max_size_help, + num2col, + put_session_message, + put_session_var, reverse_coordinates, + update_data ) from ishtar_common.data_importer import ( Importer, @@ -237,7 +239,7 @@ class ImporterType(models.Model): return col_number = 1 # user number so we start with 1 lst_col_number = 0 - for column in self.columns.order_by("col_number").all(): + for column in self.columns.filter(col_number__gt=0).order_by("col_number").all(): while column.col_number > col_number: col_number += 1 # header @@ -291,10 +293,13 @@ class ImporterType(models.Model): DEFAULTS = dict( [(default.keys, default.values) for default in self.defaults.all()] ) + PRE_IMPORT_VALUES = {} + if import_instance: + PRE_IMPORT_VALUES = import_instance.pre_import_values LINE_FORMAT = [] LINE_EXPORT_FORMAT = [] idx = 0 - for column in self.columns.order_by("col_number").all(): + for column in self.columns.filter(col_number__gt=0).order_by("col_number").all(): idx += 1 while column.col_number > idx: LINE_FORMAT.append(None) @@ -354,6 +359,7 @@ class ImporterType(models.Model): "OBJECT_CLS": OBJECT_CLS, "DESC": self.description, "DEFAULTS": DEFAULTS, + "PRE_IMPORT_VALUES": PRE_IMPORT_VALUES, "LINE_FORMAT": LINE_FORMAT, "UNICITY_KEYS": UNICITY_KEYS, "LINE_EXPORT_FORMAT": LINE_EXPORT_FORMAT, @@ -1129,6 +1135,26 @@ class FormaterType(models.Model): def natural_key(self): return self.formater_type, self.options, self.many_split + @property + def associated_model(self): + if hasattr(self, "__associated_model"): + return self.__associated_model + if self.formater_type != "TypeFormater" or not self.options: + self.__associated_model = None + return + self.__associated_model = None + options = self.options.split(".") + if len(options) == 1: + app = "ishtar_common" + else: + app = options[0] + model_name = options[-1] + try: + self.__associated_model = apps.get_model(app, model_name) + except LookupError: + pass + return self.__associated_model + def __str__(self): return " - ".join( [ @@ -1331,6 +1357,14 @@ class BaseImport(models.Model): class Meta: abstract = True + @property + def has_pre_import_form(self) -> bool: + raise NotImplemented() + + @property + def pre_import_form_is_valid(self) -> bool: + raise NotImplemented() + class ImportGroup(BaseImport): importer_type = models.ForeignKey(ImporterGroup, on_delete=models.CASCADE, @@ -1353,6 +1387,10 @@ class ImportGroup(BaseImport): return False @property + def pre_import_form_is_valid(self) -> bool: + return not any(-1 for imp in self.imports.all() if not imp.pre_import_form_is_valid) + + @property def import_id(self): return f"group-{self.id}" @@ -1386,7 +1424,8 @@ class ImportGroup(BaseImport): actions.append(("A", _("Analyse"))) if self.state == "A": actions.append(("A", _("Re-analyse"))) - actions.append(("I", _("Launch import"))) + if not any(-1 for imp in self.import_list() if not imp.pre_import_form_is_valid): + actions.append(("I", _("Launch import"))) if self.state in ("F", "FE"): actions.append(("A", _("Re-analyse"))) actions.append(("I", _("Re-import"))) @@ -1614,6 +1653,52 @@ class Import(BaseImport): return False return True + @property + def pre_import_values(self) -> dict: + values = {} + for column in self.importer_type.columns.filter(col_number__lte=0): + q = ImportColumnValue.objects.filter(column=column, import_item=self) + if not q.count(): + continue + column_value = q.all()[0] + q = column_value.column.targets.all() + if not q.count(): + continue + target = q.all()[0] + + value = column_value.value + if value == "": + continue + many = target.formater_type.many_split + if many: + if value.startswith("['") and value.endswith("']"): + value = value[2:-2].split("', '") + associated_model = target.formater_type.associated_model + if associated_model: + if many: + try: + value = [associated_model.objects.get(pk=int(pk)) for pk in value] + except (associated_model.DoesNotExist, ValueError): + continue + else: + try: + value = associated_model.objects.get(pk=value) + except (associated_model.DoesNotExist, ValueError): + continue + + keys = target.target.split("__") + dct = generate_dict_from_list(keys, value) + values = update_data(values, dct) + + q = column_value.column.duplicate_fields.all() + if not q.count(): + continue + for dup in q.all(): + keys = dup.field_name.split("__") + dct = generate_dict_from_list(keys, value) + values = update_data(values, dct) + return values + def get_number_of_lines(self): if self.number_of_line: return self.number_of_line @@ -1704,7 +1789,8 @@ class Import(BaseImport): actions.append(("A", _("Analyse"))) if self.state in ("A", "PI"): actions.append(("A", _("Re-analyse"))) - actions.append(("I", _("Launch import"))) + if self.pre_import_form_is_valid: + actions.append(("I", _("Launch import"))) if profile.experimental_feature: if self.changed_checked: actions.append(("IS", _("Step by step import"))) diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py index 621e96b11..4e50c506c 100644 --- a/ishtar_common/utils.py +++ b/ishtar_common/utils.py @@ -274,32 +274,45 @@ def check_model_access_control(request, model, available_perms=None): return allowed, own -def update_data(data_1, data_2, merge=False): +def update_data(data, new_data, merge=False): """ Update a data directory taking account of key detail """ res = {} - if not isinstance(data_1, dict) or not isinstance(data_2, dict): - if data_2 and not data_1: - return data_2 + if not isinstance(data, dict) or not isinstance(new_data, dict): + if new_data and not data: + return new_data if not merge: - if data_2: - return data_2 - return data_1 - if data_2 and data_2 != data_1: - return data_1 + " ; " + data_2 - return data_1 - for k in data_1: - if k not in data_2: - res[k] = data_1[k] + if new_data: + return new_data + return data + if new_data and data_2 != data: + return data + " ; " + new_data + return data + for k in data: + if k not in new_data: + res[k] = data[k] else: - res[k] = update_data(data_1[k], data_2[k], merge=merge) - for k in data_2: - if k not in data_1: - res[k] = data_2[k] + res[k] = update_data(data[k], new_data[k], merge=merge) + for k in new_data: + if k not in data: + res[k] = new_data[k] return res +def generate_dict_from_list(lst: list, value) -> dict: + """ + ("key1", "key2", "key3"), value -> {"key1": {"key2": {"key3": value}}} + """ + dct = {} + for key in reversed(lst): + if not dct: + dct = {key: value} + else: + dct = {key: dct} + return dct + + def move_dict_data(data, key1, key2): """ Move key1 value to key2 value in a data dict |