Import: add GeoRSS import

8 files changed, 102 insertions, 7 deletions

diff --git a/chimere/models.py b/chimere/models.py
index ee2d43e..56f4055 100644
--- a/chimere/models.py
+++ b/chimere/models.py
@@ -41,7 +41,8 @@ from django.utils.translation import ugettext_lazy as _
 from chimere.widgets import PointField, RouteField, SelectMultipleField, \
                             TextareaWidget, DatePickerWidget
 from chimere.managers import BaseGeoManager
-from chimere.utils import KMLManager, OSMManager, ShapefileManager
+from chimere.utils import KMLManager, OSMManager, ShapefileManager, \
+                          GeoRSSManager
 
 class News(models.Model):
     """News of the site
@@ -204,12 +205,14 @@ class SubCategory(models.Model):
 
 IMPORTERS = {'KML':KMLManager,
              'OSM':OSMManager,
-             'SHP':ShapefileManager
+             'SHP':ShapefileManager,
+             'RSS':GeoRSSManager
              }
 
 IMPORTER_CHOICES = (('KML', 'KML'),
                     ('OSM', 'OSM'),
                     ('SHP', 'Shapefile'),
+                    ('RSS', 'GeoRSS')
                     )
 
 IMPORTER_CHOICES_DICT = dict(IMPORTER_CHOICES)
diff --git a/chimere/tests.py b/chimere/tests.py
index b3ad3fb..78655ea 100644
--- a/chimere/tests.py
+++ b/chimere/tests.py
@@ -152,6 +152,15 @@ class OSMImporterTest(TestCase, ImporterTest):
 
         self.marker_importers = [(importer1, 19), (importer2, 8)]
 
+class GeoRSSImporterTest(TestCase, ImporterTest):
+    def setUp(self):
+        subcategory_1, subcategory_2 = subcategory_setup()
+        importer1 = Importer.objects.create(importer_type='RSS',
+                         source=test_dir_path+'tests/georss.xml')
+        importer1.categories.add(subcategory_1)
+
+        self.marker_importers = [(importer1, 1)]
+
 class FeedsTest(TestCase):
     def setUp(self):
         self.area = area_setup()
diff --git a/chimere/tests/georss.xml b/chimere/tests/georss.xml
new file mode 100644
index 0000000..8697f16
--- /dev/null
+++ b/chimere/tests/georss.xml
@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom" 
+  xmlns:georss="http://www.georss.org/georss">
+  <title>Earthquakes</title>
+  <subtitle>International earthquake observation labs</subtitle>
+  <link href="http://example.org/"/>
+  <updated>2005-12-13T18:30:02Z</updated>
+  <author>
+    <name>Dr. Thaddeus Remor</name>
+    <email>tremor@quakelab.edu</email>
+  </author>
+  <id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
+  <entry>
+    <title>M 3.2, Mona Passage</title>
+    <link href="http://example.org/2005/09/09/atom01"/>
+    <id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
+    <updated>2005-08-17T07:02:32Z</updated>
+    <summary>We just had a big one.</summary>
+    <georss:point>45.256 -71.92</georss:point>
+  </entry>
+</feed>
diff --git a/chimere/utils.py b/chimere/utils.py
index 9398488..fb478dc 100644
--- a/chimere/utils.py
+++ b/chimere/utils.py
@@ -23,6 +23,7 @@ Utilitaries
 
 import csv
 import datetime
+import feedparser
 import os
 import re
 import StringIO
@@ -71,6 +72,7 @@ class ImportManager:
 
     def create_or_update_item(self, cls, values, import_key, version=None):
         updated, created, item = False, False, None
+        import_key = unicode(import_key).replace(':', '^')
         if import_key:
             dct_import = {
                 'import_key__icontains':'%s:%s;' % (
@@ -475,6 +477,63 @@ class CSVManager(ImportManager):
         result = render_to_response('chimere/export_%s.csv' % cls_name, dct)
         return filename, result
 
+class GeoRSSManager(ImportManager):
+    u"""
+    RSS importer.
+    This manager only gets and do not produce GeoRSSFeed
+    """
+
+    def get(self):
+        u"""
+        Get data from a GeoRSS simple source
+
+        Return a tuple with:
+         - number of new item ;
+         - number of item updated ;
+         - error detail on error
+        """
+        from models import Marker
+        new_item, updated_item, msg = 0, 0, ''
+        feed = feedparser.parse(self.importer_instance.source)
+        if feed['bozo']:
+            return (0, 0, _(u"RSS feed is not well formed"))
+        for item in feed['items']:
+            if "georss_point" not in item and 'georss_line' not in item:
+                continue
+            cls = None
+            dct = {'origin':self.importer_instance.origin,
+                   'license':self.importer_instance.license}
+            if 'georss_point' in item:
+                cls = Marker
+                try:
+                    y, x = item['georss_point'].split(' ')
+                except ValueError:
+                    continue
+                dct['point'] = 'SRID=4326;POINT(%s %s)' % (x, y)
+                for k in ['description', 'summary', 'value']:
+                    if k in item:
+                        dct['description'] = item[k]
+                        break
+            else:
+                cls = Route
+                points = item['georss_line'].split(' ')
+                reordered_points = []
+                # lat, lon -> x, y
+                for idx in xrange(len(points)/2):
+                    reordered_points.append(points[idx*2+1])
+                    reordered_points.append(points[idx*2])
+                dct['route'] = 'SRID=4326;LINESTRING(%s)' % \
+                            ",".join(reordered_points)
+
+            dct['name'] = item['title']
+            pl_id = item['id'] if 'id' in item else item['title']
+            it, updated, created = self.create_or_update_item(cls, dct, pl_id)
+            if updated:
+                updated_item += 1
+            if created:
+                new_item += 1
+        return (new_item, updated_item, msg)
+
 RE_HOOK = re.compile('\[([^\]]*)\]')
 
 # TODO: manage deleted item from OSM
diff --git a/debian/control b/debian/control
index 1c93646..7d3a110 100644
--- a/debian/control
+++ b/debian/control
@@ -5,6 +5,6 @@ Depends: python-django (>=1.4), python-gdal, python-psycopg2,
          libjs-jquery-ui, libjs-jquery-ui-theme-base,
          postgresql-9.1, postgresql-9.1-postgis, gettext,
          python-simplejson, gpsbabel, python-django-south,
-         python-pyexiv2
+         python-pyexiv2 python-feedparser
 Recommends: tinymce, python-django-celery, python-kombu
 Suggests: libjs-jquery-ui-theme-south-street
diff --git a/docs/install.rst b/docs/install.rst
index e1cb458..a6d3558 100644
--- a/docs/install.rst
+++ b/docs/install.rst
@@ -37,6 +37,7 @@ Prerequisites
  - `lxml <http://lxml.de/>`_
  - `jquery <http://jquery.com/>`_ version 1.7.1 or better
  - `jquery-ui <http://jqueryui.com/>`_
+ - `Universal Feed Parser <https://code.google.com/p/feedparser/>`_
 
 geodjango is a part of django since version 1.0 but it has some specific
 (geographically related) additional dependencies:
@@ -62,7 +63,7 @@ Linux distribution repositories. For instance on Debian Wheezy::
         python-gdal python-lxml python-psycopg2 python-imaging gettext   \
         postgresql-9.1 postgresql-9.1-postgis libjs-jquery libjs-jquery-ui \
         python-django-celery python-simplejson python-gdal gpsbabel \
-        python-pyexiv2
+        python-pyexiv2 python-feedparser
 
 On Debian Squeeze (you need to activate backports)::
 
@@ -72,7 +73,7 @@ On Debian Squeeze (you need to activate backports)::
         apache2 libgeos-3.2.0 proj-bin gdal-bin python-gdal python-lxml \
         python-psycopg2 python-imaging gettext postgresql-8.4 \
         postgresql-8.4-postgis libjs-jquery libjs-jquery-ui python-simplejson \
-        python-gdal gpsbabel python-pyexiv2
+        python-gdal gpsbabel python-pyexiv2 python-feedparser
 
 
 If these packages do not exist in your distribution's repository, please refer
diff --git a/docs/upgrade.rst b/docs/upgrade.rst
index 65e3734..b697978 100644
--- a/docs/upgrade.rst
+++ b/docs/upgrade.rst
@@ -44,7 +44,8 @@ Activate the backports: http://backports-master.debian.org/Instructions/
 Then install the new dependencies::
 
     apt-get install -t squeeze-backports python-django python-django-south \
-                       python-simplejson libjs-jquery-ui python-pyexiv2
+                       python-simplejson libjs-jquery-ui python-pyexiv2 \
+                       python-feedparser
 
 Debian Wheezy
 +++++++++++++
@@ -52,7 +53,7 @@ Debian Wheezy
 .. code-block:: bash
 
     apt-get install python-django-south python-simplejson libjs-jquery-ui \
-                    python-pyexiv2
+                    python-pyexiv2 python-feedparser
 
 If you are planing to do major import consider the install of `Celery
 <http://celeryproject.org/>`_.
diff --git a/requirements.txt b/requirements.txt
index 843a351..7c073dd 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,3 +6,4 @@ lxml
 south==0.7.3
 simplejson
 unidecode
+feedparser