Step by step imports: general check of lines to filter only relevant lines (refs #3975)

1 files changed, 152 insertions, 17 deletions

diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index 9afd435da..84008bb01 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -760,16 +760,19 @@ class FormaterType(models.Model):
             return IMPORTER_TYPES_DCT[self.formater_type](**kwargs)
 
 
-IMPORT_STATE = (("C", _(u"Created")),
-                ("AP", _(u"Analyse in progress")),
-                ("A", _(u"Analysed")),
-                ("P", _(u"Import pending")),
-                ("IQ", _(u"Import in queue")),
-                ("IP", _(u"Import in progress")),
-                ("FE", _(u"Finished with errors")),
-                ("F", _(u"Finished")),
-                ("AC", _(u"Archived")),
-                )
+IMPORT_STATE = (
+    ("C", _(u"Created")),
+    ("AP", _(u"Analyse in progress")),
+    ("A", _(u"Analysed")),
+    ("HQ", _(u"Check modified in queue")),
+    ("IQ", _(u"Import in queue")),
+    ("HP", _(u"Check modified in progress")),
+    ("IP", _(u"Import in progress")),
+    ("PI", _(u"Partially imported")),
+    ("FE", _(u"Finished with errors")),
+    ("F", _(u"Finished")),
+    ("AC", _(u"Archived")),
+)
 
 IMPORT_STATE_DCT = dict(IMPORT_STATE)
 ENCODINGS = [(settings.ENCODING, settings.ENCODING),
@@ -777,6 +780,7 @@ ENCODINGS = [(settings.ENCODING, settings.ENCODING),
              ('utf-8', 'utf-8')]
 
 delayed_import = None
+delayed_check = None
 
 
 if settings.USE_BACKGROUND_TASK:
@@ -788,6 +792,14 @@ if settings.USE_BACKGROUND_TASK:
             pass
         imp.importation(session_key=session_key)
 
+    @background(schedule=1)
+    def delayed_check(import_pk, session_key):
+        try:
+            imp = Import.objects.get(pk=import_pk)
+        except Import.DoesNotExist:
+            pass
+        imp.check_modified(session_key=session_key)
+
 
 class Import(models.Model):
     user = models.ForeignKey('IshtarUser', blank=True, null=True,
@@ -838,6 +850,12 @@ class Import(models.Model):
         _(u"Imported line numbers"), blank=True, null=True,
         validators=[validate_comma_separated_integer_list]
     )
+    changed_checked = models.BooleanField(_(u"Changed have been checked"),
+                                          default=False)
+    changed_line_numbers = models.TextField(
+        _(u"Changed line numbers"), blank=True, null=True,
+        validators=[validate_comma_separated_integer_list]
+    )
 
     class Meta:
         verbose_name = _(u"Import")
@@ -879,7 +897,7 @@ class Import(models.Model):
 
     def add_imported_line(self, idx_line):
         if self.imported_line_numbers and \
-                idx_line in self.imported_line_numbers.split(','):
+                str(idx_line) in self.imported_line_numbers.split(','):
             return
         if self.imported_line_numbers:
             self.imported_line_numbers += ","
@@ -888,6 +906,35 @@ class Import(models.Model):
         self.imported_line_numbers += str(idx_line)
         self.save()
 
+    def add_changed_line(self, idx_line):
+        if self.changed_line_numbers and \
+                str(idx_line) in self.changed_line_numbers.split(','):
+            return
+        if self.changed_line_numbers:
+            self.changed_line_numbers += ","
+        else:
+            self.changed_line_numbers = ""
+        self.changed_line_numbers += str(idx_line)
+        self.save()
+
+    def remove_changed_line(self, idx_line):
+        if not self.changed_line_numbers:
+            return
+        line_numbers = self.changed_line_numbers.split(',')
+        if str(idx_line) not in line_numbers:
+            return
+        line_numbers.pop(line_numbers.index(str(idx_line)))
+        self.changed_line_numbers = ",".join(line_numbers)
+        self.save()
+
+    def has_changes(self, idx_line):
+        if not self.changed_checked:
+            return True
+        if not self.changed_line_numbers:
+            return
+        line_numbers = self.changed_line_numbers.split(',')
+        return str(idx_line) in line_numbers
+
     def line_is_imported(self, idx_line):
         return self.imported_line_numbers and \
                str(idx_line) in self.imported_line_numbers.split(',')
@@ -901,16 +948,24 @@ class Import(models.Model):
         actions = []
         if self.state == 'C':
             actions.append(('A', _(u"Analyse")))
-        if self.state == 'A':
+        if self.state in ('A', 'PI'):
             actions.append(('A', _(u"Re-analyse")))
             actions.append(('I', _(u"Launch import")))
             if profile.experimental_feature:
-                actions.append(('IS', _(u"Step by step import")))
+                if self.changed_checked:
+                    actions.append(('IS', _(u"Step by step import")))
+                    actions.append(('CH', _(u"Re-check for changes")))
+                else:
+                    actions.append(('CH', _(u"Check for changes")))
         if self.state in ('F', 'FE'):
             actions.append(('A', _(u"Re-analyse")))
             actions.append(('I', _(u"Re-import")))
             if profile.experimental_feature:
-                actions.append(('IS', _(u"Step by step import")))
+                if self.changed_checked:
+                    actions.append(('IS', _(u"Step by step re-import")))
+                    actions.append(('CH', _(u"Re-check for changes")))
+                else:
+                    actions.append(('CH', _(u"Check for changes")))
             actions.append(('AC', _(u"Archive")))
         if self.state == 'AC':
             actions.append(('A', _(u"Unarchive")))
@@ -977,6 +1032,77 @@ class Import(models.Model):
         self.end_date = datetime.datetime.now()
         self.save()
 
+    def delayed_check_modified(self, session_key):
+        if not settings.USE_BACKGROUND_TASK:
+            return self.check_modified(session_key=session_key)
+        put_session_message(
+            session_key,
+            unicode(
+                _(u"Modification check {} added to the queue")).format(
+                self.name),
+            "info")
+        self.state = 'HQ'
+        self.end_date = datetime.datetime.now()
+        self.save()
+        return delayed_check(self.pk, session_key)
+
+    def check_modified(self, session_key=None):
+        self.state = 'HP'
+        self.end_date = datetime.datetime.now()
+        self.changed_line_numbers = ""
+        self.changed_checked = False
+        self.save()
+
+        for idx in range(self.skip_lines, self.get_number_of_lines() + 1):
+            try:
+                imprt, data = self.importation(
+                    simulate=True,
+                    line_to_process=idx,
+                    return_importer_and_data=True
+                )
+            except IOError as e:
+                # error is identified as a change
+                self.add_changed_line(idx)
+                continue
+
+            # no data is not normal and an error is identified as a change
+            if not data or not data[0]:
+                self.add_changed_line(idx)
+                continue
+
+            # new objects is a change
+            if imprt.new_objects:
+                self.add_changed_line(idx)
+                continue
+
+            # check all updated fields
+            changed = False
+            for path, obj, values, updated_values in imprt.updated_objects:
+                if changed:
+                    break
+                for k in updated_values.keys():
+                    if changed:
+                        break
+                    current_value = getattr(obj, k)
+                    updated_value = updated_values[k]
+                    if hasattr(current_value, 'all'):
+                        current_value = list(current_value.all())
+                        changed = False
+                        for v in updated_value:
+                            if v not in current_value:
+                                changed = True
+                                break
+                    else:
+                        if current_value != updated_value:
+                            changed = True
+                            break
+            if changed:
+                self.add_changed_line(idx)
+                continue
+            self.remove_changed_line(idx)
+        self.changed_checked = True
+        self.save()
+
     def delayed_importation(self, session_key):
         if not settings.USE_BACKGROUND_TASK:
             return self.importation(session_key=session_key)
@@ -1010,7 +1136,10 @@ class Import(models.Model):
                     ids = []
                 ids.append(self.pk)
                 put_session_var(session_key, 'current_import_id', ids)
-            self.state = 'FE'
+            if line_to_process:
+                self.state = 'PI'
+            else:
+                self.state = 'FE'
             self.save()
             if not return_importer_and_data:
                 return
@@ -1023,7 +1152,10 @@ class Import(models.Model):
             result_file, ContentFile(importer.get_csv_result().encode('utf-8')))
 
         if importer.errors:
-            self.state = 'FE'
+            if line_to_process:
+                self.state = 'PI'
+            else:
+                self.state = 'FE'
             error_file = filename + "_errors_%s.csv" % now
             self.error_file.save(
                 error_file,
@@ -1033,7 +1165,10 @@ class Import(models.Model):
                 self.name)
             msg_cls = "warning"
         else:
-            self.state = 'F'
+            if line_to_process:
+                self.state = 'PI'
+            else:
+                self.state = 'F'
             self.error_file = None
             msg = unicode(_(u"Import {} finished with no errors")).format(
                 self.name)