summaryrefslogtreecommitdiff
path: root/ishtar_common/data_importer.py
diff options
context:
space:
mode:
authorÉtienne Loks <etienne.loks@iggdrasil.net>2023-08-10 13:48:59 +0200
committerÉtienne Loks <etienne.loks@iggdrasil.net>2023-08-10 13:48:59 +0200
commitd5769edc5764517b01523081b4dcbbffe7731c7d (patch)
treece9e8909e80d0e1608900e8101c4c1966db8808c /ishtar_common/data_importer.py
parent7f98a4e7d80f7ebba5568447db9bdef33c37c54f (diff)
downloadIshtar-d5769edc5764517b01523081b4dcbbffe7731c7d.tar.bz2
Ishtar-d5769edc5764517b01523081b4dcbbffe7731c7d.zip
⚰️ remove old code on imports
Diffstat (limited to 'ishtar_common/data_importer.py')
-rw-r--r--ishtar_common/data_importer.py227
1 files changed, 2 insertions, 225 deletions
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index e8fe4e3c0..ae4cb3675 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -493,25 +493,6 @@ class StrChoiceFormater(Formater, ChoiceChecker):
):
self.init_db_target(user)
- """
- from ishtar_common.models import TargetKey
- if self.db_target:
- q = {'target': self.db_target,
- 'associated_import': import_instance,
- 'is_set': True
- }
- for v in self.equiv_dict:
- q['key'] = v
- value = self.equiv_dict[v]
- if hasattr(value, 'pk'):
- value = value.pk
- q['value'] = value
- with transaction.atomic():
- try:
- t, created = TargetKey.objects.get_or_create(**q)
- except IntegrityError:
- pass
- """
if (not output or output == "silent") and not choose_default:
return
if self.many_split:
@@ -522,60 +503,10 @@ class StrChoiceFormater(Formater, ChoiceChecker):
values = new_values
TargetKey = apps.get_model("ishtar_common", "TargetKey")
for value in set(values):
- base_value = copy.copy(value)
value = self.prepare(value)
if value in self.equiv_dict:
continue
- if output != "cli" and not choose_default:
- self.missings.add(value)
- continue
- msgstr, idx = self._get_choices(comment)
- res = None
- if choose_default:
- res = 1
- while res not in range(1, idx + 1):
- msg = msgstr % value
- sys.stdout.write(msg.encode("utf-8"))
- sys.stdout.write("\n>>> ")
- res = input()
- try:
- res = int(res)
- except ValueError:
- pass
- res -= 1
- if res < len(self.choices):
- v = self.choices[res][0]
- if self.model and v:
- v = self.model.objects.get(pk=v)
- self.equiv_dict[value] = v
- self.add_key(v, value, import_instance)
- self.new_keys[value] = v
- elif self.create and res == len(self.choices):
- self.equiv_dict[value] = self.new(base_value)
- self.choices.append(
- (self.equiv_dict[value].pk, str(self.equiv_dict[value]))
- )
- self.new_keys[value] = str(self.equiv_dict[value])
- else:
- self.equiv_dict[value] = None
- if self.equiv_dict[value] and self.db_target:
- q = {"target": self.db_target, "key": value}
- query = TargetKey.objects.filter(**q)
- query = query.filter(self._base_target_filter(user))
- if query.count():
- target = query.all()[0]
- target.value = self.equiv_dict[value]
- target.is_set = True
- target.save()
- else:
- q["associated_import"] = import_instance
- with transaction.atomic():
- q["value"] = self.equiv_dict[value]
- q["is_set"] = True
- try:
- TargetKey.objects.create(**q)
- except IntegrityError:
- pass
+ self.missings.add(value)
if output == "db" and self.db_target:
for missing in self.missings:
q = {"target": self.db_target, "key": missing}
@@ -597,8 +528,6 @@ class StrChoiceFormater(Formater, ChoiceChecker):
TargetKey.objects.create(**q)
except IntegrityError:
pass
- if output == "cli":
- self.report_new(comment)
def new(self, value):
return
@@ -766,39 +695,11 @@ class StrToBoolean(Formater, ChoiceChecker):
):
if (not output or output == "silent") and not choose_default:
return
- msgstr = comment + " - "
- msgstr += str(
- _('Choice for "%s" is not available. ' "Which one is relevant?\n")
- )
- msgstr += "1. True\n"
- msgstr += "2. False\n"
- msgstr += "3. Empty\n"
for value in values:
value = self.prepare(value)
if value in self.dct:
continue
- if output != "cli" and not choose_default:
- self.missings.add(value)
- continue
- res = None
- if choose_default:
- res = 1
- while res not in range(1, 4):
- msg = msgstr % value
- sys.stdout.write(msg.encode("utf-8"))
- sys.stdout.write("\n>>> ")
- res = input()
- try:
- res = int(res)
- except ValueError:
- pass
- if res == 1:
- self.dct[value] = True
- elif res == 2:
- self.dct[value] = False
- else:
- self.dct[value] = None
- self.new_keys[value] = str(self.dct[value])
+ self.missings.add(value)
if output == "db" and self.db_target:
TargetKey = apps.get_model("ishtar_common", "TargetKey")
@@ -813,8 +714,6 @@ class StrToBoolean(Formater, ChoiceChecker):
TargetKey.objects.create(**q)
except IntegrityError:
pass
- if output == "cli":
- self.report_new(comment)
def format(self, value):
origin_value = value
@@ -876,102 +775,6 @@ class Importer(object):
)
}
- def _create_models(self, force=False):
- """
- Create a db config from a hardcoded import.
- Not useful anymore?
- """
- from ishtar_common import models
-
- q = models.ImporterType.objects.filter(slug=self.SLUG)
- if not force and (not self.SLUG or q.count()):
- return
- if force and q.count():
- q.all()[0].delete()
- name = self.NAME if self.NAME else self.SLUG
-
- model_name = self.OBJECT_CLS.__module__ + "." + self.OBJECT_CLS.__name__
- model_cls, c = models.ImporterModel.object.get_or_create(
- klass=model_name, default={"name": self.OBJECT_CLS.__name__}
- )
-
- unicity_keys = ""
- if self.UNICITY_KEYS:
- unicity_keys = ";".join(self.UNICITY_KEYS)
-
- importer = models.ImporterType.objects.create(
- slug=self.SLUG,
- name=name,
- description=self.DESC,
- associated_models=model_cls,
- unicity_keys=unicity_keys,
- )
-
- for default in self.DEFAULTS:
- values = self.DEFAULTS[default]
- imp_default = models.ImporterDefault.objects.create(
- importer_type=importer, target="__".join(default)
- )
- for key in values:
- if key in ("history_modifier",):
- continue
- value = values[key]
- if hasattr(value, "txt_idx") and value.txt_idx:
- value = value.txt_idx
- elif hasattr(value, "pk") and value.pk:
- value = value.pk
- if callable(value):
- value = value()
- models.ImporterDefaultValues.objects.create(
- default_target=imp_default, target=key, value=value
- )
- for idx, line in enumerate(self.line_format):
- idx += 1
- if not line:
- continue
- column = models.ImporterColumn.objects.create(
- importer_type=importer, col_number=idx
- )
- targets = line.field_name
- if type(targets) not in (list, tuple):
- targets = [targets]
- formaters = line.formater
- if type(formaters) not in (list, tuple):
- formaters = [formaters]
- for idx_target, target in enumerate(targets):
- formater = formaters[idx_target]
- formater_name = formater.__class__.__name__
- if formater_name not in models.IMPORTER_TYPES_DCT:
- formater_name = "UnknowType"
- options = ""
- if formater_name == "TypeFormater":
- options = formater.model.__module__ + "." + formater.model.__name__
- elif formater_name == "UnicodeFormater":
- options = str(formater.max_length or "")
- elif formater_name == "DateFormater":
- options = formater.date_formats[0]
- formater_model, created = models.FormaterType.objects.get_or_create(
- formater_type=formater_name,
- options=options.strip(),
- many_split=getattr(formater, "many_split", None) or "",
- )
- regexp_filter = None
- if getattr(formater, "regexp", None):
- regexp_filter, created = models.Regexp.objects.get_or_create(
- regexp=formater.regex, defaults={"name": "Default name"}
- )
- models.ImportTarget.objects.get_or_create(
- column=column,
- target=target,
- formater_type=formater_model,
- force_new=getattr(formater, "force_new", False),
- concat=getattr(formater, "concat", False),
- concat_str=getattr(formater, "concat_str", ""),
- regexp_filter=regexp_filter,
- comment=line.comment,
- )
- return True
-
def _get_improperly_conf_error(self, model):
cls_name = str(model._meta.verbose_name)
return ImporterError(str(self.ERRORS["improperly_configured"]).format(cls_name))
@@ -1163,32 +966,6 @@ class Importer(object):
def get_current_values(self, obj):
return obj
- def _associate_db_target_to_formaters(self):
- if not self.import_instance:
- return
- self.DB_TARGETS = {}
- from ishtar_common.models import ImporterColumn, ImportTarget
-
- for idx, line in enumerate(self.line_format):
- idx += 1
- if not line:
- continue
- col = ImporterColumn.objects.get(
- importer_type=self.import_instance.importer_type, col_number=idx
- )
- formater = line.formater
- targets = line.field_name
- if type(formater) not in (list, tuple):
- formater = [formater]
- targets = [targets]
- for target in targets:
- tg = target
- if type(target) == list and type(target[0]) == list:
- tg = target[0]
- self.DB_TARGETS["{}-{}".format(idx, tg)] = ImportTarget.objects.get(
- column=col, target=tg
- )
-
@classmethod
def _field_name_to_data_dict(
cls,