diff options
| -rw-r--r-- | ishtar_common/data_importer.py | 64 | ||||
| -rw-r--r-- | ishtar_common/management/commands/ishtar_import.py | 15 | ||||
| -rw-r--r-- | ishtar_common/models_imports.py | 4 |
3 files changed, 60 insertions, 23 deletions
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index 46658f43d..0572a30a5 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -44,6 +44,7 @@ from ishtar_common.utils import ( get_current_profile, get_file_from_link, update_data, + BColors ) @@ -898,14 +899,25 @@ class Importer(object): self.errors.append((idx_line, None, msg)) return item - def post_import(self): + def post_import(self, verbose=False): related_list = {} if self.import_instance and not self.line_to_process: self.import_instance.state = "PP" self.import_instance.imported_line_numbers = "" self.import_instance.save() self.timestamp = int(datetime.datetime.now().timestamp()) - for cls_pk, idx_line in self.post_save_items.items(): + items = self.post_save_items.items() + start = datetime.datetime.now() + total = len(items) + for cls_pk, idx_line in items: + if verbose: + txt = BColors.OKBLUE + f"\r\t- post-import: {idx_line + 1}/{total}" + left = self._get_eta(idx_line, total, start) + if left: + txt += f" ({left} seconds left)" + txt += BColors.ENDC + sys.stdout.write(txt) + sys.stdout.flush() if self.import_instance and not self.line_to_process: self.import_instance.add_imported_line(idx_line) cls, pk = cls_pk @@ -923,6 +935,12 @@ class Importer(object): if hasattr(item, "fix"): # post save/m2m specific fix item.fix() + if verbose: + elapsed = datetime.datetime.now() - start + txt = BColors.OKBLUE + f"\r\t- import: {total} items post treated in {elapsed}\n" + txt += BColors.ENDC + sys.stdout.write(txt) + sys.stdout.flush() for cls, pk in related_list.keys(): try: item = cls.objects.get(pk=pk) @@ -990,6 +1008,7 @@ class Importer(object): user=None, line_to_process=None, simulate=False, + verbose=False ): if initialize: self.initialize( @@ -997,7 +1016,7 @@ class Importer(object): ) self.simulate = simulate self.line_to_process = line_to_process - return self._importation(table) + return self._importation(table, verbose=verbose) def get_current_values(self, obj): return obj @@ -1059,7 +1078,16 @@ class Importer(object): current_data = current_data[key] return data - def _importation(self, table): + def _get_eta(self, idx_line, total, start): + left = None + if idx_line > 10: + elapsed = datetime.datetime.now() - start + time_by_item = elapsed / idx_line + if time_by_item: + left = ((total - idx_line) * time_by_item).seconds + return left + + def _importation(self, table, verbose=False): self.match_table = {} table = list(table) if not table or not table[0]: @@ -1093,35 +1121,33 @@ class Importer(object): self.now = datetime.datetime.now() start = datetime.datetime.now() total = len(table) - if self.output == "cli": - sys.stdout.write("\n") results = [] for idx_line, line in enumerate(table): self.idx_line = idx_line + if verbose: + left = self._get_eta(idx_line, total, start) + txt = BColors.OKBLUE + f"\r\t- import: {idx_line + 1}/{total}" + if left: + txt += f" ({left} seconds left)" + sys.stdout.write(txt + BColors.ENDC) + sys.stdout.flush() if self.line_to_process is not None: if self.line_to_process != idx_line: continue if idx_line > self.line_to_process: return results - if self.output == "cli": - left = None - if idx_line > 10: - ellapsed = datetime.datetime.now() - start - time_by_item = ellapsed / idx_line - if time_by_item: - left = ((total - idx_line) * time_by_item).seconds - txt = "\r* %d/%d" % (idx_line + 1, total) - if left: - txt += " (%d seconds left)" % left - sys.stdout.write(txt.encode("utf-8")) - sys.stdout.flush() try: results.append(self._line_processing(idx_line, line)) except ImporterError as import_error: msg = str(import_error) if not any(1 for error in self.IGNORE_ERRORS if error in msg): self.errors.append((idx_line, None, msg)) - self.post_import() + if verbose: + elapsed = datetime.datetime.now() - start + txt = BColors.OKBLUE + f"\r\t- import: {total} items imported in {elapsed}\n" + sys.stdout.write(txt + BColors.ENDC) + sys.stdout.flush() + self.post_import(verbose=verbose) for item in self.to_be_close: item.close() return results diff --git a/ishtar_common/management/commands/ishtar_import.py b/ishtar_common/management/commands/ishtar_import.py index 011af2f8c..3698cfa98 100644 --- a/ishtar_common/management/commands/ishtar_import.py +++ b/ishtar_common/management/commands/ishtar_import.py @@ -4,6 +4,7 @@ from django.core.management.base import BaseCommand, CommandError from ishtar_common import models, models_imports +from ishtar_common.utils import BColors class Command(BaseCommand): @@ -16,10 +17,14 @@ class Command(BaseCommand): parser.add_argument('command', choices=["list", "analyse", "import", "archive"]) parser.add_argument('import_id', nargs='?', default=None) + parser.add_argument( + "--quiet", dest="quiet", action="store_true", help="Quiet output" + ) def handle(self, *args, **options): command = options['command'] import_id = options['import_id'] + quiet = options.get("quiet", False) if command != "list" and not import_id: raise CommandError("With {} <import_id> is mandatory".format( command)) @@ -39,7 +44,7 @@ class Command(BaseCommand): self.stdout.flush() return try: - imp = models.Import.objects.get(pk=args[1]) + imp = models.Import.objects.get(pk=import_id) except (ValueError, models.Import.DoesNotExist): raise CommandError("{} is not a valid import ID".format(args[0])) if command == 'analyse': @@ -47,9 +52,11 @@ class Command(BaseCommand): self.stdout.write("* {} analysed\n".format(imp)) self.stdout.flush() elif command == 'import': - self.stdout.write("* import {}\n".format(imp)) - imp.importation() - self.stdout.write("* {} imported\n".format(imp)) + if not quiet: + self.stdout.write(BColors.OKGREEN + f"* import {imp}{BColors.ENDC}\n") + imp.importation(verbose=not quiet) + if not quiet: + self.stdout.write(BColors.OKGREEN + f"* {imp} imported{BColors.ENDC}\n") self.stdout.flush() elif command == 'archive': imp.archive() diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py index 2b181165b..836676c03 100644 --- a/ishtar_common/models_imports.py +++ b/ishtar_common/models_imports.py @@ -2622,6 +2622,7 @@ class Import(BaseImport): simulate=False, return_importer_and_data=False, request=None, + verbose=False ): self.state = "IP" self.end_date = datetime.datetime.now() @@ -2636,6 +2637,7 @@ class Import(BaseImport): user=self.user, line_to_process=line_to_process, simulate=simulate, + verbose=verbose ) except IOError: error_message = str(_("Error on imported file: {}")).format( @@ -2802,6 +2804,8 @@ class Import(BaseImport): if not self.error_file or not self.error_file.path: ImportLineError.objects.filter(import_item=self).delete() return + if not os.path.isfile(self.error_file.path): + return with open(self.error_file.path, "r") as error_file: reader = csv.reader(error_file) for idx, line in enumerate(reader): |