From fc88c946b177981dd2b8f26c0474abfa3c2e2647 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Fri, 6 Apr 2018 13:38:16 +0200 Subject: Step by step imports: general check of lines to filter only relevant lines (refs #3975) --- ishtar_common/data_importer.py | 5 + .../migrations/0039_auto_20180405_1923.py | 32 ++++ ishtar_common/models_imports.py | 169 ++++++++++++++++++--- .../templates/ishtar/import_step_by_step.html | 30 +++- ishtar_common/urls.py | 3 + ishtar_common/views.py | 76 ++++++--- 6 files changed, 277 insertions(+), 38 deletions(-) create mode 100644 ishtar_common/migrations/0039_auto_20180405_1923.py diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index 3ee173a1f..0caccf46d 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -1098,6 +1098,11 @@ class Importer(object): if not line: self.validity.append([]) return + if not self.simulate and self.import_instance and \ + not self.import_instance.has_changes(idx_line): + self.validity.append(line) + return + self._throughs = [] # list of (formater, value) self._post_processing = [] # list of (formater, value) self._item_post_processing = [] diff --git a/ishtar_common/migrations/0039_auto_20180405_1923.py b/ishtar_common/migrations/0039_auto_20180405_1923.py new file mode 100644 index 000000000..770bfb9aa --- /dev/null +++ b/ishtar_common/migrations/0039_auto_20180405_1923.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-04-05 19:23 +from __future__ import unicode_literals + +import django.core.validators +from django.db import migrations, models +import re + + +class Migration(migrations.Migration): + + dependencies = [ + ('ishtar_common', '0038_auto_20180403_1130'), + ] + + operations = [ + migrations.AddField( + model_name='import', + name='changed_checked', + field=models.BooleanField(default=False, verbose_name='Changed have been checked'), + ), + migrations.AddField( + model_name='import', + name='changed_line_numbers', + field=models.TextField(blank=True, null=True, validators=[django.core.validators.RegexValidator(re.compile('^\\d+(?:\\,\\d+)*\\Z'), code='invalid', message='Enter only digits separated by commas.')], verbose_name='Changed line numbers'), + ), + migrations.AlterField( + model_name='import', + name='state', + field=models.CharField(choices=[(b'C', 'Created'), (b'AP', 'Analyse in progress'), (b'A', 'Analysed'), (b'HQ', 'Check modified in queue'), (b'IQ', 'Import in queue'), (b'HP', 'Check modified in progress'), (b'IP', 'Import in progress'), (b'FE', 'Finished with errors'), (b'F', 'Finished'), (b'AC', 'Archived')], default='C', max_length=2, verbose_name='State'), + ), + ] diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py index 9afd435da..84008bb01 100644 --- a/ishtar_common/models_imports.py +++ b/ishtar_common/models_imports.py @@ -760,16 +760,19 @@ class FormaterType(models.Model): return IMPORTER_TYPES_DCT[self.formater_type](**kwargs) -IMPORT_STATE = (("C", _(u"Created")), - ("AP", _(u"Analyse in progress")), - ("A", _(u"Analysed")), - ("P", _(u"Import pending")), - ("IQ", _(u"Import in queue")), - ("IP", _(u"Import in progress")), - ("FE", _(u"Finished with errors")), - ("F", _(u"Finished")), - ("AC", _(u"Archived")), - ) +IMPORT_STATE = ( + ("C", _(u"Created")), + ("AP", _(u"Analyse in progress")), + ("A", _(u"Analysed")), + ("HQ", _(u"Check modified in queue")), + ("IQ", _(u"Import in queue")), + ("HP", _(u"Check modified in progress")), + ("IP", _(u"Import in progress")), + ("PI", _(u"Partially imported")), + ("FE", _(u"Finished with errors")), + ("F", _(u"Finished")), + ("AC", _(u"Archived")), +) IMPORT_STATE_DCT = dict(IMPORT_STATE) ENCODINGS = [(settings.ENCODING, settings.ENCODING), @@ -777,6 +780,7 @@ ENCODINGS = [(settings.ENCODING, settings.ENCODING), ('utf-8', 'utf-8')] delayed_import = None +delayed_check = None if settings.USE_BACKGROUND_TASK: @@ -788,6 +792,14 @@ if settings.USE_BACKGROUND_TASK: pass imp.importation(session_key=session_key) + @background(schedule=1) + def delayed_check(import_pk, session_key): + try: + imp = Import.objects.get(pk=import_pk) + except Import.DoesNotExist: + pass + imp.check_modified(session_key=session_key) + class Import(models.Model): user = models.ForeignKey('IshtarUser', blank=True, null=True, @@ -838,6 +850,12 @@ class Import(models.Model): _(u"Imported line numbers"), blank=True, null=True, validators=[validate_comma_separated_integer_list] ) + changed_checked = models.BooleanField(_(u"Changed have been checked"), + default=False) + changed_line_numbers = models.TextField( + _(u"Changed line numbers"), blank=True, null=True, + validators=[validate_comma_separated_integer_list] + ) class Meta: verbose_name = _(u"Import") @@ -879,7 +897,7 @@ class Import(models.Model): def add_imported_line(self, idx_line): if self.imported_line_numbers and \ - idx_line in self.imported_line_numbers.split(','): + str(idx_line) in self.imported_line_numbers.split(','): return if self.imported_line_numbers: self.imported_line_numbers += "," @@ -888,6 +906,35 @@ class Import(models.Model): self.imported_line_numbers += str(idx_line) self.save() + def add_changed_line(self, idx_line): + if self.changed_line_numbers and \ + str(idx_line) in self.changed_line_numbers.split(','): + return + if self.changed_line_numbers: + self.changed_line_numbers += "," + else: + self.changed_line_numbers = "" + self.changed_line_numbers += str(idx_line) + self.save() + + def remove_changed_line(self, idx_line): + if not self.changed_line_numbers: + return + line_numbers = self.changed_line_numbers.split(',') + if str(idx_line) not in line_numbers: + return + line_numbers.pop(line_numbers.index(str(idx_line))) + self.changed_line_numbers = ",".join(line_numbers) + self.save() + + def has_changes(self, idx_line): + if not self.changed_checked: + return True + if not self.changed_line_numbers: + return + line_numbers = self.changed_line_numbers.split(',') + return str(idx_line) in line_numbers + def line_is_imported(self, idx_line): return self.imported_line_numbers and \ str(idx_line) in self.imported_line_numbers.split(',') @@ -901,16 +948,24 @@ class Import(models.Model): actions = [] if self.state == 'C': actions.append(('A', _(u"Analyse"))) - if self.state == 'A': + if self.state in ('A', 'PI'): actions.append(('A', _(u"Re-analyse"))) actions.append(('I', _(u"Launch import"))) if profile.experimental_feature: - actions.append(('IS', _(u"Step by step import"))) + if self.changed_checked: + actions.append(('IS', _(u"Step by step import"))) + actions.append(('CH', _(u"Re-check for changes"))) + else: + actions.append(('CH', _(u"Check for changes"))) if self.state in ('F', 'FE'): actions.append(('A', _(u"Re-analyse"))) actions.append(('I', _(u"Re-import"))) if profile.experimental_feature: - actions.append(('IS', _(u"Step by step import"))) + if self.changed_checked: + actions.append(('IS', _(u"Step by step re-import"))) + actions.append(('CH', _(u"Re-check for changes"))) + else: + actions.append(('CH', _(u"Check for changes"))) actions.append(('AC', _(u"Archive"))) if self.state == 'AC': actions.append(('A', _(u"Unarchive"))) @@ -977,6 +1032,77 @@ class Import(models.Model): self.end_date = datetime.datetime.now() self.save() + def delayed_check_modified(self, session_key): + if not settings.USE_BACKGROUND_TASK: + return self.check_modified(session_key=session_key) + put_session_message( + session_key, + unicode( + _(u"Modification check {} added to the queue")).format( + self.name), + "info") + self.state = 'HQ' + self.end_date = datetime.datetime.now() + self.save() + return delayed_check(self.pk, session_key) + + def check_modified(self, session_key=None): + self.state = 'HP' + self.end_date = datetime.datetime.now() + self.changed_line_numbers = "" + self.changed_checked = False + self.save() + + for idx in range(self.skip_lines, self.get_number_of_lines() + 1): + try: + imprt, data = self.importation( + simulate=True, + line_to_process=idx, + return_importer_and_data=True + ) + except IOError as e: + # error is identified as a change + self.add_changed_line(idx) + continue + + # no data is not normal and an error is identified as a change + if not data or not data[0]: + self.add_changed_line(idx) + continue + + # new objects is a change + if imprt.new_objects: + self.add_changed_line(idx) + continue + + # check all updated fields + changed = False + for path, obj, values, updated_values in imprt.updated_objects: + if changed: + break + for k in updated_values.keys(): + if changed: + break + current_value = getattr(obj, k) + updated_value = updated_values[k] + if hasattr(current_value, 'all'): + current_value = list(current_value.all()) + changed = False + for v in updated_value: + if v not in current_value: + changed = True + break + else: + if current_value != updated_value: + changed = True + break + if changed: + self.add_changed_line(idx) + continue + self.remove_changed_line(idx) + self.changed_checked = True + self.save() + def delayed_importation(self, session_key): if not settings.USE_BACKGROUND_TASK: return self.importation(session_key=session_key) @@ -1010,7 +1136,10 @@ class Import(models.Model): ids = [] ids.append(self.pk) put_session_var(session_key, 'current_import_id', ids) - self.state = 'FE' + if line_to_process: + self.state = 'PI' + else: + self.state = 'FE' self.save() if not return_importer_and_data: return @@ -1023,7 +1152,10 @@ class Import(models.Model): result_file, ContentFile(importer.get_csv_result().encode('utf-8'))) if importer.errors: - self.state = 'FE' + if line_to_process: + self.state = 'PI' + else: + self.state = 'FE' error_file = filename + "_errors_%s.csv" % now self.error_file.save( error_file, @@ -1033,7 +1165,10 @@ class Import(models.Model): self.name) msg_cls = "warning" else: - self.state = 'F' + if line_to_process: + self.state = 'PI' + else: + self.state = 'F' self.error_file = None msg = unicode(_(u"Import {} finished with no errors")).format( self.name) diff --git a/ishtar_common/templates/ishtar/import_step_by_step.html b/ishtar_common/templates/ishtar/import_step_by_step.html index 9aca80837..998bf99c6 100644 --- a/ishtar_common/templates/ishtar/import_step_by_step.html +++ b/ishtar_common/templates/ishtar/import_step_by_step.html @@ -2,15 +2,37 @@ {% load i18n inline_formset link_to_window %} {% block content %} +

+ {% trans "Back to import list" %} +

+ + +