Step by step imports: general check of lines to filter only relevant lines (refs #3975)

6 files changed, 277 insertions, 38 deletions

diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index 3ee173a1f..0caccf46d 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -1098,6 +1098,11 @@ class Importer(object):
         if not line:
             self.validity.append([])
             return
+        if not self.simulate and self.import_instance and \
+                not self.import_instance.has_changes(idx_line):
+            self.validity.append(line)
+            return
+
         self._throughs = []  # list of (formater, value)
         self._post_processing = []  # list of (formater, value)
         self._item_post_processing = []
diff --git a/ishtar_common/migrations/0039_auto_20180405_1923.py b/ishtar_common/migrations/0039_auto_20180405_1923.py
new file mode 100644
index 000000000..770bfb9aa
--- /dev/null
+++ b/ishtar_common/migrations/0039_auto_20180405_1923.py
@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.10 on 2018-04-05 19:23
+from __future__ import unicode_literals
+
+import django.core.validators
+from django.db import migrations, models
+import re
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ishtar_common', '0038_auto_20180403_1130'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='import',
+            name='changed_checked',
+            field=models.BooleanField(default=False, verbose_name='Changed have been checked'),
+        ),
+        migrations.AddField(
+            model_name='import',
+            name='changed_line_numbers',
+            field=models.TextField(blank=True, null=True, validators=[django.core.validators.RegexValidator(re.compile('^\\d+(?:\\,\\d+)*\\Z'), code='invalid', message='Enter only digits separated by commas.')], verbose_name='Changed line numbers'),
+        ),
+        migrations.AlterField(
+            model_name='import',
+            name='state',
+            field=models.CharField(choices=[(b'C', 'Created'), (b'AP', 'Analyse in progress'), (b'A', 'Analysed'), (b'HQ', 'Check modified in queue'), (b'IQ', 'Import in queue'), (b'HP', 'Check modified in progress'), (b'IP', 'Import in progress'), (b'FE', 'Finished with errors'), (b'F', 'Finished'), (b'AC', 'Archived')], default='C', max_length=2, verbose_name='State'),
+        ),
+    ]
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index 9afd435da..84008bb01 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -760,16 +760,19 @@ class FormaterType(models.Model):
             return IMPORTER_TYPES_DCT[self.formater_type](**kwargs)
 
 
-IMPORT_STATE = (("C", _(u"Created")),
-                ("AP", _(u"Analyse in progress")),
-                ("A", _(u"Analysed")),
-                ("P", _(u"Import pending")),
-                ("IQ", _(u"Import in queue")),
-                ("IP", _(u"Import in progress")),
-                ("FE", _(u"Finished with errors")),
-                ("F", _(u"Finished")),
-                ("AC", _(u"Archived")),
-                )
+IMPORT_STATE = (
+    ("C", _(u"Created")),
+    ("AP", _(u"Analyse in progress")),
+    ("A", _(u"Analysed")),
+    ("HQ", _(u"Check modified in queue")),
+    ("IQ", _(u"Import in queue")),
+    ("HP", _(u"Check modified in progress")),
+    ("IP", _(u"Import in progress")),
+    ("PI", _(u"Partially imported")),
+    ("FE", _(u"Finished with errors")),
+    ("F", _(u"Finished")),
+    ("AC", _(u"Archived")),
+)
 
 IMPORT_STATE_DCT = dict(IMPORT_STATE)
 ENCODINGS = [(settings.ENCODING, settings.ENCODING),
@@ -777,6 +780,7 @@ ENCODINGS = [(settings.ENCODING, settings.ENCODING),
              ('utf-8', 'utf-8')]
 
 delayed_import = None
+delayed_check = None
 
 
 if settings.USE_BACKGROUND_TASK:
@@ -788,6 +792,14 @@ if settings.USE_BACKGROUND_TASK:
             pass
         imp.importation(session_key=session_key)
 
+    @background(schedule=1)
+    def delayed_check(import_pk, session_key):
+        try:
+            imp = Import.objects.get(pk=import_pk)
+        except Import.DoesNotExist:
+            pass
+        imp.check_modified(session_key=session_key)
+
 
 class Import(models.Model):
     user = models.ForeignKey('IshtarUser', blank=True, null=True,
@@ -838,6 +850,12 @@ class Import(models.Model):
         _(u"Imported line numbers"), blank=True, null=True,
         validators=[validate_comma_separated_integer_list]
     )
+    changed_checked = models.BooleanField(_(u"Changed have been checked"),
+                                          default=False)
+    changed_line_numbers = models.TextField(
+        _(u"Changed line numbers"), blank=True, null=True,
+        validators=[validate_comma_separated_integer_list]
+    )
 
     class Meta:
         verbose_name = _(u"Import")
@@ -879,7 +897,7 @@ class Import(models.Model):
 
     def add_imported_line(self, idx_line):
         if self.imported_line_numbers and \
-                idx_line in self.imported_line_numbers.split(','):
+                str(idx_line) in self.imported_line_numbers.split(','):
             return
         if self.imported_line_numbers:
             self.imported_line_numbers += ","
@@ -888,6 +906,35 @@ class Import(models.Model):
         self.imported_line_numbers += str(idx_line)
         self.save()
 
+    def add_changed_line(self, idx_line):
+        if self.changed_line_numbers and \
+                str(idx_line) in self.changed_line_numbers.split(','):
+            return
+        if self.changed_line_numbers:
+            self.changed_line_numbers += ","
+        else:
+            self.changed_line_numbers = ""
+        self.changed_line_numbers += str(idx_line)
+        self.save()
+
+    def remove_changed_line(self, idx_line):
+        if not self.changed_line_numbers:
+            return
+        line_numbers = self.changed_line_numbers.split(',')
+        if str(idx_line) not in line_numbers:
+            return
+        line_numbers.pop(line_numbers.index(str(idx_line)))
+        self.changed_line_numbers = ",".join(line_numbers)
+        self.save()
+
+    def has_changes(self, idx_line):
+        if not self.changed_checked:
+            return True
+        if not self.changed_line_numbers:
+            return
+        line_numbers = self.changed_line_numbers.split(',')
+        return str(idx_line) in line_numbers
+
     def line_is_imported(self, idx_line):
         return self.imported_line_numbers and \
                str(idx_line) in self.imported_line_numbers.split(',')
@@ -901,16 +948,24 @@ class Import(models.Model):
         actions = []
         if self.state == 'C':
             actions.append(('A', _(u"Analyse")))
-        if self.state == 'A':
+        if self.state in ('A', 'PI'):
             actions.append(('A', _(u"Re-analyse")))
             actions.append(('I', _(u"Launch import")))
             if profile.experimental_feature:
-                actions.append(('IS', _(u"Step by step import")))
+                if self.changed_checked:
+                    actions.append(('IS', _(u"Step by step import")))
+                    actions.append(('CH', _(u"Re-check for changes")))
+                else:
+                    actions.append(('CH', _(u"Check for changes")))
         if self.state in ('F', 'FE'):
             actions.append(('A', _(u"Re-analyse")))
             actions.append(('I', _(u"Re-import")))
             if profile.experimental_feature:
-                actions.append(('IS', _(u"Step by step import")))
+                if self.changed_checked:
+                    actions.append(('IS', _(u"Step by step re-import")))
+                    actions.append(('CH', _(u"Re-check for changes")))
+                else:
+                    actions.append(('CH', _(u"Check for changes")))
             actions.append(('AC', _(u"Archive")))
         if self.state == 'AC':
             actions.append(('A', _(u"Unarchive")))
@@ -977,6 +1032,77 @@ class Import(models.Model):
         self.end_date = datetime.datetime.now()
         self.save()
 
+    def delayed_check_modified(self, session_key):
+        if not settings.USE_BACKGROUND_TASK:
+            return self.check_modified(session_key=session_key)
+        put_session_message(
+            session_key,
+            unicode(
+                _(u"Modification check {} added to the queue")).format(
+                self.name),
+            "info")
+        self.state = 'HQ'
+        self.end_date = datetime.datetime.now()
+        self.save()
+        return delayed_check(self.pk, session_key)
+
+    def check_modified(self, session_key=None):
+        self.state = 'HP'
+        self.end_date = datetime.datetime.now()
+        self.changed_line_numbers = ""
+        self.changed_checked = False
+        self.save()
+
+        for idx in range(self.skip_lines, self.get_number_of_lines() + 1):
+            try:
+                imprt, data = self.importation(
+                    simulate=True,
+                    line_to_process=idx,
+                    return_importer_and_data=True
+                )
+            except IOError as e:
+                # error is identified as a change
+                self.add_changed_line(idx)
+                continue
+
+            # no data is not normal and an error is identified as a change
+            if not data or not data[0]:
+                self.add_changed_line(idx)
+                continue
+
+            # new objects is a change
+            if imprt.new_objects:
+                self.add_changed_line(idx)
+                continue
+
+            # check all updated fields
+            changed = False
+            for path, obj, values, updated_values in imprt.updated_objects:
+                if changed:
+                    break
+                for k in updated_values.keys():
+                    if changed:
+                        break
+                    current_value = getattr(obj, k)
+                    updated_value = updated_values[k]
+                    if hasattr(current_value, 'all'):
+                        current_value = list(current_value.all())
+                        changed = False
+                        for v in updated_value:
+                            if v not in current_value:
+                                changed = True
+                                break
+                    else:
+                        if current_value != updated_value:
+                            changed = True
+                            break
+            if changed:
+                self.add_changed_line(idx)
+                continue
+            self.remove_changed_line(idx)
+        self.changed_checked = True
+        self.save()
+
     def delayed_importation(self, session_key):
         if not settings.USE_BACKGROUND_TASK:
             return self.importation(session_key=session_key)
@@ -1010,7 +1136,10 @@ class Import(models.Model):
                     ids = []
                 ids.append(self.pk)
                 put_session_var(session_key, 'current_import_id', ids)
-            self.state = 'FE'
+            if line_to_process:
+                self.state = 'PI'
+            else:
+                self.state = 'FE'
             self.save()
             if not return_importer_and_data:
                 return
@@ -1023,7 +1152,10 @@ class Import(models.Model):
             result_file, ContentFile(importer.get_csv_result().encode('utf-8')))
 
         if importer.errors:
-            self.state = 'FE'
+            if line_to_process:
+                self.state = 'PI'
+            else:
+                self.state = 'FE'
             error_file = filename + "_errors_%s.csv" % now
             self.error_file.save(
                 error_file,
@@ -1033,7 +1165,10 @@ class Import(models.Model):
                 self.name)
             msg_cls = "warning"
         else:
-            self.state = 'F'
+            if line_to_process:
+                self.state = 'PI'
+            else:
+                self.state = 'F'
             self.error_file = None
             msg = unicode(_(u"Import {} finished with no errors")).format(
                 self.name)
diff --git a/ishtar_common/templates/ishtar/import_step_by_step.html b/ishtar_common/templates/ishtar/import_step_by_step.html
index 9aca80837..998bf99c6 100644
--- a/ishtar_common/templates/ishtar/import_step_by_step.html
+++ b/ishtar_common/templates/ishtar/import_step_by_step.html
@@ -2,15 +2,37 @@
 {% load i18n inline_formset link_to_window %}
 
 {% block content %}
+<p class="text-center">
+    <a href="{% url 'current_imports' %}">{% trans "Back to import list" %}</a>
+</p>
+
+<nav class="row">
+    <ul class="col-md-12 justify-content-center nav nav-pills">
+      <li class="nav-item">
+          <a class="nav-link{% if not all %} active{% endif %}"
+              href="{% url 'import_step_by_step' import.pk line_number %}">
+              {% trans "Add/modified" %}
+          </a>
+      </li>
+      <li class="nav-item">
+          <a class="nav-link{% if all %} active{% endif %}"
+             href="{% url 'import_step_by_step_all' import.pk line_number %}">
+              {% trans "All lines" %}
+          </a>
+      </li>
+    </ul>
+</nav>
+
 
 <nav class="row">
     <ul class="pagination col-md-6 justify-content-center">
-    {% for label, page_number, enabled, imported in page_numbers %}
+    {% for label, page_number, enabled, imported, changed in page_numbers %}
         <li class="page-item{% if not enabled %} disabled{% endif %}"
             {% if not enabled %} tabindex="-1"{% endif %}>
-            <a class="page-link{% if imported %} imported-page{% endif %}{% if page_number == line_number_displayed %} current-page{% endif %}"
-               {% if imported %}title="{% trans 'Already imported' %}"{% endif %}
-               href="{% url 'import_step_by_step' import.pk page_number %}">
+            <a class="page-link{% if imported or not changed %} imported-page{% endif %}{% if page_number == line_number_displayed %} current-page{% endif %}"
+               {% if imported %}title="{% trans 'Already imported' %}"{% else %}
+               {% if not changed %}title="{% trans 'No changes' %}"{% endif %}{% endif %}
+               href="{% url import_url import.pk page_number %}">
                 {{label}}
             </a>
         </li>
diff --git a/ishtar_common/urls.py b/ishtar_common/urls.py
index 1a7051b17..538041b82 100644
--- a/ishtar_common/urls.py
+++ b/ishtar_common/urls.py
@@ -117,6 +117,9 @@ urlpatterns = [
         views.ImportDeleteView.as_view(), name='import_delete'),
     url(r'^import-link-unmatched/(?P<pk>[0-9]+)/$',
         views.ImportLinkView.as_view(), name='import_link_unmatched'),
+    url(r'^import-step-by-step/all/(?P<pk>[0-9]+)/(?P<line_number>[0-9]+)/$',
+        views.ImportStepByStepView.as_view(), name='import_step_by_step_all',
+        kwargs={'all_pages': True}),
     url(r'^import-step-by-step/(?P<pk>[0-9]+)/(?P<line_number>[0-9]+)/$',
         views.ImportStepByStepView.as_view(), name='import_step_by_step'),
 ]
diff --git a/ishtar_common/views.py b/ishtar_common/views.py
index d4fb68004..2c2a6fe7e 100644
--- a/ishtar_common/views.py
+++ b/ishtar_common/views.py
@@ -1822,6 +1822,11 @@ class ImportListView(IshtarMixin, LoginRequiredMixin, ListView):
                     imprt.delayed_importation(request.session.session_key)
                 else:
                     imprt.importation()
+            elif action == 'CH':
+                if settings.USE_BACKGROUND_TASK:
+                    imprt.delayed_check_modified(request.session.session_key)
+                else:
+                    imprt.check_modified()
             elif action == 'IS':
                 if imprt.current_line is None:
                     imprt.current_line = imprt.skip_lines
@@ -1944,35 +1949,67 @@ class ImportStepByStepView(IshtarMixin, LoginRequiredMixin, TemplateView):
         return super(ImportStepByStepView, self).get(request, *args, **kwargs)
 
     def get_pagination(self, dct):
-        pagination_step = 10
+        pagination_step = 5
+        only_modified = not self.kwargs.get('all_pages', False)
+        dct['all'] = not only_modified
+        dct['import_url'] = 'import_step_by_step' if only_modified else \
+            'import_step_by_step_all'
         line_nb = self.imprt_obj.get_number_of_lines()
         total_line_nb = self.imprt_obj.skip_lines + line_nb
-        delta = int((self.current_line_number + 1) / pagination_step)
+        delta = 0
 
         already_imported = []
         if self.imprt_obj.imported_line_numbers:
             already_imported = self.imprt_obj.imported_line_numbers.split(',')
+        changes = []
+        if self.imprt_obj.changed_line_numbers:
+            changes = self.imprt_obj.changed_line_numbers.split(',')
 
         dct['page_is_last'] = self.current_line_number == line_nb
-        dct['page_numbers'] = []  # label, number, enabled, is_imported
-        if delta > 0:
-            dct['page_numbers'].append(
-                (_(u"Previous"), delta * pagination_step - 1, True, False)
+        # label, number, enabled, is_imported, has_changes
+        dct['page_numbers'] = []
+        # first pass for the delta
+        current = 0
+        for idx in range(self.imprt_obj.skip_lines, total_line_nb):
+            imported = str(idx) in already_imported
+            changed = str(idx) in changes
+            if only_modified and (imported or not changed):
+                continue
+            current += 1
+            if idx == self.current_line_number - 1:
+                delta = int(current / pagination_step)
+
+        current, has_next, previous = 0, False, None
+        for idx in range(self.imprt_obj.skip_lines, total_line_nb):
+            if current >= ((delta + 1) * pagination_step):
+                has_next = idx
+                break
+            imported = str(idx) in already_imported
+            changed = str(idx) in changes
+            if only_modified and (imported or not changed):
+                continue
+            current += 1
+            if current <= (delta * pagination_step):
+                previous = idx
+                continue
+            nb = idx + 1
+            dct['page_numbers'].append((nb, nb, True, imported, changed))
+
+        if previous:
+            dct['page_numbers'].insert(0,
+                (_(u"Previous"), previous + 1, True, False,
+                 True)
             )
         else:
-            dct['page_numbers'].append(
-                (_(u"Previous"), self.imprt_obj.skip_lines, False, False)
+            dct['page_numbers'].insert(0,
+                (_(u"Previous"), self.imprt_obj.skip_lines, False, False, True)
             )
-        for idx in range(0, 10):
-            nb = idx + delta * pagination_step
-            if nb > self.imprt_obj.skip_lines and nb <= total_line_nb:
-                imported = str(nb) in already_imported
-                dct['page_numbers'].append((nb, nb, True, imported))
-        if nb + 1 <= total_line_nb:
-            dct['page_numbers'].append((_(u"Next"), nb + 1, True, False))
+        if has_next:
+            dct['page_numbers'].append((_(u"Next"), has_next + 1, True, False,
+                                        True))
         else:
             dct['page_numbers'].append((_(u"Next"), total_line_nb, False,
-                                        False))
+                                        False, True))
 
     def get_context_data(self, **kwargs):
         dct = super(ImportStepByStepView, self).get_context_data(**kwargs)
@@ -2107,7 +2144,12 @@ class ImportStepByStepView(IshtarMixin, LoginRequiredMixin, TemplateView):
                                                            old_and_updated)
             updated_objects.append((label, obj, values, old_and_updated))
 
-        dct['have_change'] = main_changed
+        dct['have_change'] = main_changed or self.imprt.new_objects
+        if dct["have_change"]:
+            self.imprt_obj.add_changed_line(self.current_line_number)
+        else:
+            self.imprt_obj.remove_changed_line(self.current_line_number)
+
         dct['updated_objects'] = []
         dct['matched_objects'] = []
         for path, obj, values, old_and_updated in updated_objects: