Fixes for not imported columns on step by step import. Tests for this feature.

3 files changed, 124 insertions, 8 deletions

diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py
index 4f9f73a43..612156df2 100644
--- a/archaeological_operations/tests.py
+++ b/archaeological_operations/tests.py
@@ -30,8 +30,8 @@ from django.core.urlresolvers import reverse
 from django.db.models import Q
 from django.test.client import Client
 
-from django.contrib.auth.models import User
-from django.contrib.auth.models import Permission
+from django.contrib.auth.models import User, Permission
+from django.utils.translation import ugettext_lazy as _
 import models
 
 from archaeological_operations import views
@@ -475,6 +475,107 @@ class ImportOperationTest(ImportTest, TestCase):
         self.assertEqual(ope2.data, {u'autre_refs': {u'arbitraire': 456}})
 
 
+class ImportStepByStepTest(ImportTest, TestCase):
+    fixtures = OPERATION_TOWNS_FIXTURES
+
+    def test_step_by_step_import(self):
+        create_user()
+
+        importer, form = self.init_ope_import()
+        impt = form.save(self.ishtar_user)
+        impt.initialize()
+        self.init_ope_targetkey(imp=impt)
+        c = Client()
+
+        first_person_nb = Person.objects.count()
+        first_ope_nb = models.Operation.objects.count()
+        import_url = reverse('import_step_by_step',
+                             kwargs={'pk': impt.pk, 'line_number': 2})
+        response = c.get(import_url)
+        # no login redirect
+        self.assertEqual(response.status_code, 302)
+        self.assertRedirects(response, "/accounts/login/?next=" + import_url)
+
+        c.login(username=self.username, password=self.password)
+        response = c.get(import_url)
+        self.assertEqual(response.status_code, 200)
+        # verify pagination for next link is OK
+        self.assertIn('href="/import-step-by-step/{}/3/"'.format(impt.pk),
+                      response.content)
+        # creation have been evaluated
+        self.assertIn(
+            unicode(_(u"New objects will be created.")),
+            response.content.decode('utf-8')
+        )
+
+        # import this line
+        posted = {"valid": "import"}
+        response = c.post(import_url, posted)
+        self.assertEqual(response.status_code, 302)
+        # successful import - go to the next line
+        new_import_url = reverse('import_step_by_step',
+                                 kwargs={'pk': impt.pk, 'line_number': 3})
+        self.assertRedirects(response, new_import_url)
+        current_ope_nb = models.Operation.objects.count()
+        self.assertEqual(current_ope_nb, first_ope_nb + 1)
+        current_person_nb = Person.objects.count()
+        self.assertEqual(current_person_nb, first_person_nb)
+
+        # re-import
+        posted = {"valid": "import"}
+        c.post(import_url, posted)
+        # no changes
+        current_ope_nb = models.Operation.objects.count()
+        self.assertEqual(current_ope_nb, first_ope_nb + 1)
+        current_person_nb = Person.objects.count()
+        self.assertEqual(current_person_nb, first_person_nb)
+
+        # on import page, already imported is visible
+        response = c.get(import_url)
+        self.assertEqual(response.status_code, 200)
+        self.assertIn(
+            unicode(_(u"This line have been already imported.")),
+            response.content.decode('utf-8')
+        )
+
+        # import next page
+        next_import_url = reverse(
+            'import_step_by_step', kwargs={'pk': impt.pk, 'line_number': 3})
+        posted = {"valid": "import"}
+        c.post(next_import_url, posted)
+        current_ope_nb = models.Operation.objects.count()
+        self.assertEqual(current_ope_nb, first_ope_nb + 2)
+        current_person_nb = Person.objects.count()
+        self.assertEqual(current_person_nb, first_person_nb + 1)
+
+        # modifiy CSV with an invalid date
+        posted = {
+            "valid": "change-csv",
+            "col-1": "4201",
+            "col-2": "Bourgogne",
+            "col-3": u"Fouille programmée",
+            "col-4": u"Oppìdum de Paris 2",
+            "col-5": u"L'opérateur",
+            "col-6": u"",
+            "col-7": "2000/01/32",
+            "col-8": "2002/12/31",
+            "col-9": "Age du Fer",
+        }
+        response = c.post(import_url, posted)
+        self.assertEqual(response.status_code, 200)
+        # modification have been made in the source CSV
+        with open(impt.imported_file.path) as fle:
+            fle.readline()  # header
+            imported_line = fle.readline()
+            self.assertIn("2000/01/32", imported_line)
+        # error detected on the source file
+        error = unicode(_(
+            u"The following error(s) has been encountered while parsing "
+            u"the source file:")
+        )
+        self.assertIn(error, response.content.decode('utf-8'))
+
+
 class ParcelTest(ImportTest, TestCase):
     fixtures = OPERATION_TOWNS_FIXTURES
 
diff --git a/ishtar_common/templates/ishtar/import_step_by_step.html b/ishtar_common/templates/ishtar/import_step_by_step.html
index 44c5389d3..9aca80837 100644
--- a/ishtar_common/templates/ishtar/import_step_by_step.html
+++ b/ishtar_common/templates/ishtar/import_step_by_step.html
@@ -85,7 +85,7 @@
             <td>{{idx}}</td>
             <td>{{name|safe}}</td>
             <td>{{raw}}</td>
-            <td><textarea name="col-{{idx}}">{{raw}}</textarea></td>
+            <td><textarea name="col-{{idx}}">{{raw|safe}}</textarea></td>
         </tr>{% endfor %}
     </table>
 
@@ -121,13 +121,18 @@
 
 {% else %}
 
-{% if not have_change %}
+{% if not have_change and not new_objects %}
 <div class="alert alert-success" role="alert">
     <p>{% trans "No change for this item." %}</p>
 </div>
 {% else %}
 <div class="alert alert-warning" role="alert">
+    {% if have_change %}
     <p>{% trans "Changes will be made for this item" %}</p>
+    {% endif %}
+    {% if new_objects %}
+    <p>{% trans "New objects will be created." %}</p>
+    {% endif %}
 </div>
 {% endif %}
 
diff --git a/ishtar_common/views.py b/ishtar_common/views.py
index 1536868d4..d4fb68004 100644
--- a/ishtar_common/views.py
+++ b/ishtar_common/views.py
@@ -1948,7 +1948,10 @@ class ImportStepByStepView(IshtarMixin, LoginRequiredMixin, TemplateView):
         line_nb = self.imprt_obj.get_number_of_lines()
         total_line_nb = self.imprt_obj.skip_lines + line_nb
         delta = int((self.current_line_number + 1) / pagination_step)
-        already_imported = self.imprt_obj.imported_line_numbers.split(',')
+
+        already_imported = []
+        if self.imprt_obj.imported_line_numbers:
+            already_imported = self.imprt_obj.imported_line_numbers.split(',')
 
         dct['page_is_last'] = self.current_line_number == line_nb
         dct['page_numbers'] = []  # label, number, enabled, is_imported
@@ -1981,7 +1984,8 @@ class ImportStepByStepView(IshtarMixin, LoginRequiredMixin, TemplateView):
         dct['errors'] = self.errors
         if self.errors:
             if self.imprt.current_csv_line:
-                headers = [f.label for f in self.imprt.get_formaters()]
+                headers = [f.label if f else _(u"Not imported")
+                           for f in self.imprt.get_formaters()]
                 dct['values'] = zip(
                     range(1, len(headers) + 1), headers,
                     self.imprt.current_csv_line
@@ -1989,6 +1993,10 @@ class ImportStepByStepView(IshtarMixin, LoginRequiredMixin, TemplateView):
             return dct
         headers, self.path_to_column, interpreted_values = [], {}, []
         for idx, formater in enumerate(self.imprt.get_formaters()):
+            if not formater:
+                headers.append(_(u"Not imported"))
+                interpreted_values.append(u"&ndash;")
+                continue
             lbl = formater.label
             if formater.comment:
                 lbl += u' <i data-toggle="tooltip" class="fa '
@@ -1997,6 +2005,7 @@ class ImportStepByStepView(IshtarMixin, LoginRequiredMixin, TemplateView):
                     formater.comment.replace(u'"', u'&quot;'))
                 lbl += u'</i>'
             headers.append(lbl)
+
             field_name = formater.field_name[0]
             if formater.export_field_name:
                 field_name = formater.export_field_name[0]
@@ -2011,16 +2020,17 @@ class ImportStepByStepView(IshtarMixin, LoginRequiredMixin, TemplateView):
                 self.path_to_column[current_field_name].append(idx)
                 field_name_tuple.pop()
 
-            for idx, key in enumerate(field_name.split('__')):
+            for idx_split, key in enumerate(field_name.split('__')):
                 if isinstance(value, dict) and key in value:
                     value = value[key]
-                elif not idx:
+                elif not idx_split:
                     # no key reach
                     value = None
                 else:
                     break
             value = self.get_value(value)
             interpreted_values.append(value)
+
         dct['values'] = zip(
             range(1, len(headers) + 1), headers, self.imprt.current_csv_line,
             interpreted_values