diff options
Diffstat (limited to 'ishtar_common/data_importer.py')
| -rw-r--r-- | ishtar_common/data_importer.py | 64 |
1 files changed, 45 insertions, 19 deletions
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index 46658f43d..0572a30a5 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -44,6 +44,7 @@ from ishtar_common.utils import ( get_current_profile, get_file_from_link, update_data, + BColors ) @@ -898,14 +899,25 @@ class Importer(object): self.errors.append((idx_line, None, msg)) return item - def post_import(self): + def post_import(self, verbose=False): related_list = {} if self.import_instance and not self.line_to_process: self.import_instance.state = "PP" self.import_instance.imported_line_numbers = "" self.import_instance.save() self.timestamp = int(datetime.datetime.now().timestamp()) - for cls_pk, idx_line in self.post_save_items.items(): + items = self.post_save_items.items() + start = datetime.datetime.now() + total = len(items) + for cls_pk, idx_line in items: + if verbose: + txt = BColors.OKBLUE + f"\r\t- post-import: {idx_line + 1}/{total}" + left = self._get_eta(idx_line, total, start) + if left: + txt += f" ({left} seconds left)" + txt += BColors.ENDC + sys.stdout.write(txt) + sys.stdout.flush() if self.import_instance and not self.line_to_process: self.import_instance.add_imported_line(idx_line) cls, pk = cls_pk @@ -923,6 +935,12 @@ class Importer(object): if hasattr(item, "fix"): # post save/m2m specific fix item.fix() + if verbose: + elapsed = datetime.datetime.now() - start + txt = BColors.OKBLUE + f"\r\t- import: {total} items post treated in {elapsed}\n" + txt += BColors.ENDC + sys.stdout.write(txt) + sys.stdout.flush() for cls, pk in related_list.keys(): try: item = cls.objects.get(pk=pk) @@ -990,6 +1008,7 @@ class Importer(object): user=None, line_to_process=None, simulate=False, + verbose=False ): if initialize: self.initialize( @@ -997,7 +1016,7 @@ class Importer(object): ) self.simulate = simulate self.line_to_process = line_to_process - return self._importation(table) + return self._importation(table, verbose=verbose) def get_current_values(self, obj): return obj @@ -1059,7 +1078,16 @@ class Importer(object): current_data = current_data[key] return data - def _importation(self, table): + def _get_eta(self, idx_line, total, start): + left = None + if idx_line > 10: + elapsed = datetime.datetime.now() - start + time_by_item = elapsed / idx_line + if time_by_item: + left = ((total - idx_line) * time_by_item).seconds + return left + + def _importation(self, table, verbose=False): self.match_table = {} table = list(table) if not table or not table[0]: @@ -1093,35 +1121,33 @@ class Importer(object): self.now = datetime.datetime.now() start = datetime.datetime.now() total = len(table) - if self.output == "cli": - sys.stdout.write("\n") results = [] for idx_line, line in enumerate(table): self.idx_line = idx_line + if verbose: + left = self._get_eta(idx_line, total, start) + txt = BColors.OKBLUE + f"\r\t- import: {idx_line + 1}/{total}" + if left: + txt += f" ({left} seconds left)" + sys.stdout.write(txt + BColors.ENDC) + sys.stdout.flush() if self.line_to_process is not None: if self.line_to_process != idx_line: continue if idx_line > self.line_to_process: return results - if self.output == "cli": - left = None - if idx_line > 10: - ellapsed = datetime.datetime.now() - start - time_by_item = ellapsed / idx_line - if time_by_item: - left = ((total - idx_line) * time_by_item).seconds - txt = "\r* %d/%d" % (idx_line + 1, total) - if left: - txt += " (%d seconds left)" % left - sys.stdout.write(txt.encode("utf-8")) - sys.stdout.flush() try: results.append(self._line_processing(idx_line, line)) except ImporterError as import_error: msg = str(import_error) if not any(1 for error in self.IGNORE_ERRORS if error in msg): self.errors.append((idx_line, None, msg)) - self.post_import() + if verbose: + elapsed = datetime.datetime.now() - start + txt = BColors.OKBLUE + f"\r\t- import: {total} items imported in {elapsed}\n" + sys.stdout.write(txt + BColors.ENDC) + sys.stdout.flush() + self.post_import(verbose=verbose) for item in self.to_be_close: item.close() return results |