Import OSM data

4 files changed, 108 insertions, 23 deletions

diff --git a/chimere/models.py b/chimere/models.py
index 22a51f9..4122712 100644
--- a/chimere/models.py
+++ b/chimere/models.py
@@ -203,6 +203,7 @@ class Importer(models.Model):
     '''
     importer_type = models.CharField(_(u"Importer type"), max_length=4,
                                      choices=IMPORTER_CHOICES)
+    # URL of a KML file or a XAPI service for OSM
     source_url = models.CharField(_(u"Source URL"), max_length=200,
                                   blank=True, null=True)
     filtr = models.CharField(_(u"Filter"), max_length=200,
diff --git a/chimere/tests.py b/chimere/tests.py
index c0a13aa..7a3960f 100644
--- a/chimere/tests.py
+++ b/chimere/tests.py
@@ -5,11 +5,13 @@ import os
 test_path = os.path.abspath(__file__)
 test_dir_path = os.path.dirname(test_path) + os.sep
 
+from django.conf import settings
 from django.test import TestCase
+
 from chimere.models import Icon, Importer, Category, SubCategory, Marker
 
-class KMLImporterTest(TestCase):
-    def setUp(self):
+class ImporterTest:
+    def _baseSetUp(self):
         category = Category.objects.create(name='Main category',
                             available=True,
                             order=1,
@@ -33,29 +35,14 @@ class KMLImporterTest(TestCase):
                             icon=icon,
                             order=1,
                             item_type='M',)
-
-        importer1 = Importer.objects.create(importer_type='KML',
-            source_url=test_dir_path+'sample.kml',
-            filtr="//kml:Folder/kml:name[text()='Category 1']/../kml:Placemark")
-        importer1.categories.add(subcategory_1)
-
-        importer2 = Importer.objects.create(importer_type='KML',
-         source_url=test_dir_path+'sample.kml',
-         filtr="//kml:Folder/kml:name[text()='Subcategory 1']/../kml:Placemark")
-        importer2.categories.add(subcategory_1)
-        importer2.categories.add(subcategory_2)
-
-        importer3 = Importer.objects.create(importer_type='KML',
-         source_url=test_path+'sample.kml',
-         filtr="//kml:Folder/kml:name[text()='Subcategory 3']/../kml:Placemark")
-        importer3.categories.add(subcategory_2)
-
-        self.marker_importers = [(importer1, 1), (importer2, 2), (importer3, 0)]
+        return [subcategory_1, subcategory_2]
 
     def test_get(self):
         nb_by_cat = {}
         for importer, awaited_nb in self.marker_importers:
             nb, nb_updated, res = importer.manager.get()
+            if awaited_nb == None:
+                continue
             self.assertEqual(nb, awaited_nb)
             self.assertEqual(nb_updated, 0)
             for cat in importer.categories.all():
@@ -69,6 +56,40 @@ class KMLImporterTest(TestCase):
         # update
         for importer, awaited_nb in self.marker_importers:
             nb, nb_updated, res = importer.manager.get()
+            if awaited_nb == None:
+                continue
             self.assertEqual(nb, 0)
             self.assertEqual(nb_updated, awaited_nb)
 
+class KMLImporterTest(TestCase, ImporterTest):
+    def setUp(self):
+        subcategory_1, subcategory_2 = self._baseSetUp()
+        importer1 = Importer.objects.create(importer_type='KML',
+            source_url=test_dir_path+'sample.kml',
+            filtr="//kml:Folder/kml:name[text()='Category 1']/../kml:Placemark")
+        importer1.categories.add(subcategory_1)
+
+        importer2 = Importer.objects.create(importer_type='KML',
+         source_url=test_dir_path+'sample.kml',
+         filtr="//kml:Folder/kml:name[text()='Subcategory 1']/../kml:Placemark")
+        importer2.categories.add(subcategory_1)
+        importer2.categories.add(subcategory_2)
+
+        importer3 = Importer.objects.create(importer_type='KML',
+         source_url=test_path+'sample.kml',
+         filtr="//kml:Folder/kml:name[text()='Subcategory 3']/../kml:Placemark")
+        importer3.categories.add(subcategory_2)
+
+        self.marker_importers = [(importer1, 1), (importer2, 2), (importer3, 0)]
+
+class OSMImporterTest(TestCase, ImporterTest):
+    def setUp(self):
+        subcategory_1, subcategory_2 = self._baseSetUp()
+        importer1 = Importer.objects.create(importer_type='OSM',
+            source_url=settings.CHIMERE_XAPI_URL,
+            filtr="node[amenity=pub]"\
+                  "[bbox=-77.041579,38.885851,-77.007247,38.900881]")
+        importer1.categories.add(subcategory_1)
+
+        self.marker_importers = [(importer1, None)]
+
diff --git a/chimere/utils.py b/chimere/utils.py
index d631fe7..e243c34 100644
--- a/chimere/utils.py
+++ b/chimere/utils.py
@@ -41,7 +41,7 @@ class ImportManager:
 
 class KMLManager(ImportManager):
     u"""
-    KML importer/exporter
+    KML importer
     The filtr argument has to be defined as a correct Xpath string pointing
     to Placemark nodes.
     A typical XPath string looks like:
@@ -70,7 +70,7 @@ class KMLManager(ImportManager):
             except IOError, msg:
                 return (new_item, updated_item, msg)
         except urllib2.URLError as error:
-            return (new_item, updated_item, error.reason)
+            return (new_item, updated_item, error.message)
         tree = etree.parse(source)
         # try to get default namespace
         if not self.ns:
@@ -118,4 +118,65 @@ class KMLManager(ImportManager):
         return (new_item, updated_item, msg)
 
 class OSMManager(ImportManager):
-    pass
+    u"""
+    OSM importer
+    The source url is a path to an OSM file or a XAPI url
+    The filtr argument is XAPI args or empty if it is an OSM file.
+    """
+
+    def get(self):
+        u"""
+        Get data from the source
+        Return a tuple with:
+        - number of new item ;
+        - number of item updated ;
+        - error detail on error
+        """
+        from models import Marker
+        new_item, updated_item, msg = 0, 0, ''
+        try:
+            source = urllib2.urlopen(self.importer_instance.source_url+
+                                     self.importer_instance.filtr)
+        except ValueError:
+            # assume it is a local file
+            try:
+                source = open(self.importer_instance.source_url)
+            except IOError, msg:
+                return (new_item, updated_item, msg)
+        except urllib2.URLError as error:
+            return (new_item, updated_item, error.message)
+        tree = etree.parse(source)
+        for node in tree.xpath('//node'):
+            name, point, linestring = None, None, None
+            node_id = node.attrib.get('id')
+            for item in node:
+                k = item.attrib.get('k')
+                if k == 'name':
+                    name = item.attrib.get('v')
+            point = 'SRID=4326;POINT(%s %s)' % (node.get('lon'),
+                                                node.get('lat'))
+            if point:
+                dct = {'point':point,
+                       'name':name,}
+                m = None
+                if node_id:
+                    dct_import = {
+                        'import_key':node_id,
+                        'import_source':self.importer_instance.source_url}
+                    try:
+                        m = Marker.objects.get(**dct_import)
+                        for k in dct:
+                            setattr(m, k, dct[k])
+                        m.save()
+                        updated_item += 1
+                    except ObjectDoesNotExist:
+                        m = None
+                        dct.update(dct_import)
+                if not m:
+                    dct['status'] = 'I'
+                    m = Marker.objects.create(**dct)
+                    new_item += 1
+                m.categories.clear()
+                for cat in self.importer_instance.categories.all():
+                    m.categories.add(cat)
+        return (new_item, updated_item, msg)
diff --git a/example_project/settings.py.example b/example_project/settings.py.example
index 9b3235b..a42a1be 100644
--- a/example_project/settings.py.example
+++ b/example_project/settings.py.example
@@ -68,6 +68,8 @@ CHIMERE_DEFAULT_CATEGORIES = [1]
 #displayOutsideMaxExtent: true, wrapDateLine: true})''' # OSM cyclemap
 CHIMERE_MAP_LAYER = "new OpenLayers.Layer.OSM.Mapnik('Mapnik')" # OSM mapnik map
 
+CHIMERE_XAPI_URL = 'http://open.mapquestapi.com/xapi/api/0.6/'
+
 DEBUG = True
 TEMPLATE_DEBUG = DEBUG