diff options
| -rw-r--r-- | chimere/models.py | 7 | ||||
| -rw-r--r-- | chimere/tests.py | 9 | ||||
| -rw-r--r-- | chimere/tests/georss.xml | 21 | ||||
| -rw-r--r-- | chimere/utils.py | 59 | ||||
| -rw-r--r-- | debian/control | 2 | ||||
| -rw-r--r-- | docs/install.rst | 5 | ||||
| -rw-r--r-- | docs/upgrade.rst | 5 | ||||
| -rw-r--r-- | requirements.txt | 1 |
8 files changed, 102 insertions, 7 deletions
diff --git a/chimere/models.py b/chimere/models.py index ee2d43e..56f4055 100644 --- a/chimere/models.py +++ b/chimere/models.py @@ -41,7 +41,8 @@ from django.utils.translation import ugettext_lazy as _ from chimere.widgets import PointField, RouteField, SelectMultipleField, \ TextareaWidget, DatePickerWidget from chimere.managers import BaseGeoManager -from chimere.utils import KMLManager, OSMManager, ShapefileManager +from chimere.utils import KMLManager, OSMManager, ShapefileManager, \ + GeoRSSManager class News(models.Model): """News of the site @@ -204,12 +205,14 @@ class SubCategory(models.Model): IMPORTERS = {'KML':KMLManager, 'OSM':OSMManager, - 'SHP':ShapefileManager + 'SHP':ShapefileManager, + 'RSS':GeoRSSManager } IMPORTER_CHOICES = (('KML', 'KML'), ('OSM', 'OSM'), ('SHP', 'Shapefile'), + ('RSS', 'GeoRSS') ) IMPORTER_CHOICES_DICT = dict(IMPORTER_CHOICES) diff --git a/chimere/tests.py b/chimere/tests.py index b3ad3fb..78655ea 100644 --- a/chimere/tests.py +++ b/chimere/tests.py @@ -152,6 +152,15 @@ class OSMImporterTest(TestCase, ImporterTest): self.marker_importers = [(importer1, 19), (importer2, 8)] +class GeoRSSImporterTest(TestCase, ImporterTest): + def setUp(self): + subcategory_1, subcategory_2 = subcategory_setup() + importer1 = Importer.objects.create(importer_type='RSS', + source=test_dir_path+'tests/georss.xml') + importer1.categories.add(subcategory_1) + + self.marker_importers = [(importer1, 1)] + class FeedsTest(TestCase): def setUp(self): self.area = area_setup() diff --git a/chimere/tests/georss.xml b/chimere/tests/georss.xml new file mode 100644 index 0000000..8697f16 --- /dev/null +++ b/chimere/tests/georss.xml @@ -0,0 +1,21 @@ +<?xml version="1.0" encoding="utf-8"?> +<feed xmlns="http://www.w3.org/2005/Atom" + xmlns:georss="http://www.georss.org/georss"> + <title>Earthquakes</title> + <subtitle>International earthquake observation labs</subtitle> + <link href="http://example.org/"/> + <updated>2005-12-13T18:30:02Z</updated> + <author> + <name>Dr. Thaddeus Remor</name> + <email>tremor@quakelab.edu</email> + </author> + <id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id> + <entry> + <title>M 3.2, Mona Passage</title> + <link href="http://example.org/2005/09/09/atom01"/> + <id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id> + <updated>2005-08-17T07:02:32Z</updated> + <summary>We just had a big one.</summary> + <georss:point>45.256 -71.92</georss:point> + </entry> +</feed> diff --git a/chimere/utils.py b/chimere/utils.py index 9398488..fb478dc 100644 --- a/chimere/utils.py +++ b/chimere/utils.py @@ -23,6 +23,7 @@ Utilitaries import csv import datetime +import feedparser import os import re import StringIO @@ -71,6 +72,7 @@ class ImportManager: def create_or_update_item(self, cls, values, import_key, version=None): updated, created, item = False, False, None + import_key = unicode(import_key).replace(':', '^') if import_key: dct_import = { 'import_key__icontains':'%s:%s;' % ( @@ -475,6 +477,63 @@ class CSVManager(ImportManager): result = render_to_response('chimere/export_%s.csv' % cls_name, dct) return filename, result +class GeoRSSManager(ImportManager): + u""" + RSS importer. + This manager only gets and do not produce GeoRSSFeed + """ + + def get(self): + u""" + Get data from a GeoRSS simple source + + Return a tuple with: + - number of new item ; + - number of item updated ; + - error detail on error + """ + from models import Marker + new_item, updated_item, msg = 0, 0, '' + feed = feedparser.parse(self.importer_instance.source) + if feed['bozo']: + return (0, 0, _(u"RSS feed is not well formed")) + for item in feed['items']: + if "georss_point" not in item and 'georss_line' not in item: + continue + cls = None + dct = {'origin':self.importer_instance.origin, + 'license':self.importer_instance.license} + if 'georss_point' in item: + cls = Marker + try: + y, x = item['georss_point'].split(' ') + except ValueError: + continue + dct['point'] = 'SRID=4326;POINT(%s %s)' % (x, y) + for k in ['description', 'summary', 'value']: + if k in item: + dct['description'] = item[k] + break + else: + cls = Route + points = item['georss_line'].split(' ') + reordered_points = [] + # lat, lon -> x, y + for idx in xrange(len(points)/2): + reordered_points.append(points[idx*2+1]) + reordered_points.append(points[idx*2]) + dct['route'] = 'SRID=4326;LINESTRING(%s)' % \ + ",".join(reordered_points) + + dct['name'] = item['title'] + pl_id = item['id'] if 'id' in item else item['title'] + it, updated, created = self.create_or_update_item(cls, dct, pl_id) + if updated: + updated_item += 1 + if created: + new_item += 1 + return (new_item, updated_item, msg) + RE_HOOK = re.compile('\[([^\]]*)\]') # TODO: manage deleted item from OSM diff --git a/debian/control b/debian/control index 1c93646..7d3a110 100644 --- a/debian/control +++ b/debian/control @@ -5,6 +5,6 @@ Depends: python-django (>=1.4), python-gdal, python-psycopg2, libjs-jquery-ui, libjs-jquery-ui-theme-base, postgresql-9.1, postgresql-9.1-postgis, gettext, python-simplejson, gpsbabel, python-django-south, - python-pyexiv2 + python-pyexiv2 python-feedparser Recommends: tinymce, python-django-celery, python-kombu Suggests: libjs-jquery-ui-theme-south-street diff --git a/docs/install.rst b/docs/install.rst index e1cb458..a6d3558 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -37,6 +37,7 @@ Prerequisites - `lxml <http://lxml.de/>`_ - `jquery <http://jquery.com/>`_ version 1.7.1 or better - `jquery-ui <http://jqueryui.com/>`_ + - `Universal Feed Parser <https://code.google.com/p/feedparser/>`_ geodjango is a part of django since version 1.0 but it has some specific (geographically related) additional dependencies: @@ -62,7 +63,7 @@ Linux distribution repositories. For instance on Debian Wheezy:: python-gdal python-lxml python-psycopg2 python-imaging gettext \ postgresql-9.1 postgresql-9.1-postgis libjs-jquery libjs-jquery-ui \ python-django-celery python-simplejson python-gdal gpsbabel \ - python-pyexiv2 + python-pyexiv2 python-feedparser On Debian Squeeze (you need to activate backports):: @@ -72,7 +73,7 @@ On Debian Squeeze (you need to activate backports):: apache2 libgeos-3.2.0 proj-bin gdal-bin python-gdal python-lxml \ python-psycopg2 python-imaging gettext postgresql-8.4 \ postgresql-8.4-postgis libjs-jquery libjs-jquery-ui python-simplejson \ - python-gdal gpsbabel python-pyexiv2 + python-gdal gpsbabel python-pyexiv2 python-feedparser If these packages do not exist in your distribution's repository, please refer diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 65e3734..b697978 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -44,7 +44,8 @@ Activate the backports: http://backports-master.debian.org/Instructions/ Then install the new dependencies:: apt-get install -t squeeze-backports python-django python-django-south \ - python-simplejson libjs-jquery-ui python-pyexiv2 + python-simplejson libjs-jquery-ui python-pyexiv2 \ + python-feedparser Debian Wheezy +++++++++++++ @@ -52,7 +53,7 @@ Debian Wheezy .. code-block:: bash apt-get install python-django-south python-simplejson libjs-jquery-ui \ - python-pyexiv2 + python-pyexiv2 python-feedparser If you are planing to do major import consider the install of `Celery <http://celeryproject.org/>`_. diff --git a/requirements.txt b/requirements.txt index 843a351..7c073dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ lxml south==0.7.3 simplejson unidecode +feedparser |