Manage XML-XSLT import

4 files changed, 28 insertions, 10 deletions

diff --git a/chimere/forms.py b/chimere/forms.py
index 1ba90a0..a57fe87 100644
--- a/chimere/forms.py
+++ b/chimere/forms.py
@@ -165,7 +165,7 @@ class ImporterAdminForm(forms.ModelForm):
            not self.cleaned_data.get('zipped'):
             raise forms.ValidationError(_(u"Shapefiles must be provided in a "\
                                           u"zipped archive."))
-        if self.cleaned_data.get('importer_type') != 'XSLT' and \
+        if self.cleaned_data.get('importer_type') not in ('XSLT', 'XXLT') and \
            self.cleaned_data.get('source') and \
            self.cleaned_data.get('source_file'):
             raise forms.ValidationError(_(u"You have to set \"source\" or "
diff --git a/chimere/models.py b/chimere/models.py
index 09987c9..3196ae6 100644
--- a/chimere/models.py
+++ b/chimere/models.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright (C) 2008-2013  Étienne Loks  <etienne.loks_AT_peacefrogsDOTnet>
+# Copyright (C) 2008-2015  Étienne Loks  <etienne.loks_AT_peacefrogsDOTnet>
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as
@@ -46,7 +46,7 @@ from chimere.widgets import HiddenPointChooserWidget, PointField, RouteField, \
                             DatePickerWidget
 from chimere.managers import BaseGeoManager
 from chimere.utils import KMLManager, OSMManager, ShapefileManager, \
-                          GeoRSSManager, CSVManager, HtmlXsltManager
+                      GeoRSSManager, CSVManager, HtmlXsltManager, XMLXsltManager
 
 class Page(models.Model):
     """Simple extra pages
@@ -304,7 +304,8 @@ IMPORTERS = {'KML':KMLManager,
              'SHP':ShapefileManager,
              'RSS':GeoRSSManager,
              'CSV':CSVManager,
-             'XSLT':HtmlXsltManager
+             'XSLT':HtmlXsltManager,
+             'XXLT':XMLXsltManager
              }
 
 IMPORTER_CHOICES = (('KML', 'KML'),
@@ -312,7 +313,8 @@ IMPORTER_CHOICES = (('KML', 'KML'),
                     ('SHP', 'Shapefile'),
                     ('RSS', 'GeoRSS'),
                     ('CSV', 'CSV'),
-                    ('XSLT', 'HTML-XSLT')
+                    ('XSLT', 'HTML-XSLT'),
+                    ('XXLT', 'XML-XSLT'),
                     )
 
 IMPORTER_CHOICES_DICT = dict(IMPORTER_CHOICES)
diff --git a/chimere/static/chimere/js/importer_interface.js b/chimere/static/chimere/js/importer_interface.js
index e15aacd..2872e12 100644
--- a/chimere/static/chimere/js/importer_interface.js
+++ b/chimere/static/chimere/js/importer_interface.js
@@ -24,6 +24,12 @@ django.jQuery(function($) {
                       'field-origin', 'field-srid', 'field-license',
                       'field-categories', 'field-overwrite',
                       'field-get_description', 'field-default_localisation',
+                      'field-automatic_update'),
+        XXLT:new Array('field-source', 'field-source_file',
+                      'field-source_file_alt', 'field-default_name',
+                      'field-origin', 'field-srid', 'field-license',
+                      'field-categories', 'field-overwrite',
+                      'field-get_description', 'field-default_localisation',
                       'field-automatic_update')
     }
     var osm_map_initialized;
@@ -55,7 +61,7 @@ django.jQuery(function($) {
                 osm_map_initialized = true;
             }
         }
-        else if (importer_val == 'XSLT'){
+        else if (importer_val == 'XSLT' || importer_val == 'XXLT'){
             $('.form-row.field-filtr').addClass('field-map');
             $('#map_edit').show();
             if (!edit_map_initialized){
diff --git a/chimere/utils.py b/chimere/utils.py
index 2d74095..f3ec751 100644
--- a/chimere/utils.py
+++ b/chimere/utils.py
@@ -85,6 +85,7 @@ class ImportManager(object):
             dct_import = {
                 'import_key__icontains':'%s:%s;' % (key, import_key),
                 'import_source':self.importer_instance.source}
+            ref_item = cls.objects.filter(**dct_import)
             try:
                 item = None
                 if pk:
@@ -878,8 +879,11 @@ DATE_PARSINGS = {'fr_FR':[
                          ]
                 }
 
+def clean_field(value):
+    return value.strip()
 
 class HtmlXsltManager(ImportManager):
+    PARSER = 'HTMLParser'
     def get(self):
         u"""
         Get data from the source
@@ -904,7 +908,7 @@ class HtmlXsltManager(ImportManager):
         # convert it to valid XHTML
         #doc, errors = tidy_document(main_page)
         doc = main_page
-        dom = etree.HTML(doc, etree.HTMLParser())
+        dom = etree.HTML(doc, getattr(etree, self.PARSER)())
         try:
             xslt = etree.parse(self.importer_instance.source_file)
             self.importer_instance.source_file.seek(0)
@@ -926,8 +930,8 @@ class HtmlXsltManager(ImportManager):
         base_url = u"/".join(self.importer_instance.source.split(u'/')[:-1])
         base_url += u"/"
         for item in newdom.getroot():
-            c_item = {child.tag:child.text for child in item.getchildren()
-                                                             if child.text}
+            c_item = {child.tag:clean_field(child.text)
+                        for child in item.getchildren() if child.text}
             # try to have more information on the linked page
             if transform_child and 'link' in c_item:
                 # not an absolute address
@@ -962,7 +966,7 @@ class HtmlXsltManager(ImportManager):
         updated_item, new_item = 0, 0
         for item in items:
             if not self.importer_instance.default_localisation and \
-               not "point" in item:
+               not "point" in item and not ("lat" in item and item['lat']):
                 continue
             cls = None
             dct = {'origin':"<a href='%s'>%s</a>" % (item['link'],
@@ -973,6 +977,9 @@ class HtmlXsltManager(ImportManager):
             if 'point' in item:
                 x, y = item['point'].split(",")
                 dct['point'] = 'SRID=4326;POINT(%s %s)' % (x, y)
+            elif 'lat' in item and item['lat']:
+                dct['point'] = 'SRID=4326;POINT(%s %s)' % (item['lon'],
+                                                           item['lat'])
             else:
                 dct['point'] = self.importer_instance.default_localisation
             dct['description'] = item['description']
@@ -1007,3 +1014,6 @@ class HtmlXsltManager(ImportManager):
             if created:
                 new_item += 1
         return (new_item, updated_item, '')
+
+class XMLXsltManager(HtmlXsltManager):
+    PARSER = 'XMLParser'