Manage XML-XSLT import

1 files changed, 14 insertions, 4 deletions

diff --git a/chimere/utils.py b/chimere/utils.py
index 2d74095..f3ec751 100644
--- a/chimere/utils.py
+++ b/chimere/utils.py
@@ -85,6 +85,7 @@ class ImportManager(object):
             dct_import = {
                 'import_key__icontains':'%s:%s;' % (key, import_key),
                 'import_source':self.importer_instance.source}
+            ref_item = cls.objects.filter(**dct_import)
             try:
                 item = None
                 if pk:
@@ -878,8 +879,11 @@ DATE_PARSINGS = {'fr_FR':[
                          ]
                 }
 
+def clean_field(value):
+    return value.strip()
 
 class HtmlXsltManager(ImportManager):
+    PARSER = 'HTMLParser'
     def get(self):
         u"""
         Get data from the source
@@ -904,7 +908,7 @@ class HtmlXsltManager(ImportManager):
         # convert it to valid XHTML
         #doc, errors = tidy_document(main_page)
         doc = main_page
-        dom = etree.HTML(doc, etree.HTMLParser())
+        dom = etree.HTML(doc, getattr(etree, self.PARSER)())
         try:
             xslt = etree.parse(self.importer_instance.source_file)
             self.importer_instance.source_file.seek(0)
@@ -926,8 +930,8 @@ class HtmlXsltManager(ImportManager):
         base_url = u"/".join(self.importer_instance.source.split(u'/')[:-1])
         base_url += u"/"
         for item in newdom.getroot():
-            c_item = {child.tag:child.text for child in item.getchildren()
-                                                             if child.text}
+            c_item = {child.tag:clean_field(child.text)
+                        for child in item.getchildren() if child.text}
             # try to have more information on the linked page
             if transform_child and 'link' in c_item:
                 # not an absolute address
@@ -962,7 +966,7 @@ class HtmlXsltManager(ImportManager):
         updated_item, new_item = 0, 0
         for item in items:
             if not self.importer_instance.default_localisation and \
-               not "point" in item:
+               not "point" in item and not ("lat" in item and item['lat']):
                 continue
             cls = None
             dct = {'origin':"<a href='%s'>%s</a>" % (item['link'],
@@ -973,6 +977,9 @@ class HtmlXsltManager(ImportManager):
             if 'point' in item:
                 x, y = item['point'].split(",")
                 dct['point'] = 'SRID=4326;POINT(%s %s)' % (x, y)
+            elif 'lat' in item and item['lat']:
+                dct['point'] = 'SRID=4326;POINT(%s %s)' % (item['lon'],
+                                                           item['lat'])
             else:
                 dct['point'] = self.importer_instance.default_localisation
             dct['description'] = item['description']
@@ -1007,3 +1014,6 @@ class HtmlXsltManager(ImportManager):
             if created:
                 new_item += 1
         return (new_item, updated_item, '')
+
+class XMLXsltManager(HtmlXsltManager):
+    PARSER = 'XMLParser'