Use overpass API for OSM imports

1 files changed, 107 insertions, 49 deletions

diff --git a/chimere/utils.py b/chimere/utils.py
index 707c6bb..a5772d7 100644
--- a/chimere/utils.py
+++ b/chimere/utils.py
@@ -39,6 +39,7 @@ import zipfile
 
 from osgeo import ogr, osr
 from osmapi import OsmApi
+import overpass
 from lxml import etree
 
 from django.conf import settings
@@ -140,9 +141,6 @@ class ImportManager(object):
             values.update({
                 'import_source': self.importer_instance.source})
             values['status'] = self.importer_instance.default_status
-            item = cls.objects.create(**values)
-            item.modified_since_import = False
-            item.save()
             try:
                 item = cls.objects.create(**values)
                 item.modified_since_import = False
@@ -766,7 +764,7 @@ class JsonManager(ImportManager):
     This manager only gets and do not produce Json feed
     """
 
-    def extract_dict_values(self, item, filtr):
+    def extract_dict_values(self, item, filtr, base_key=None):
         """
         Extract values from a dict.
 
@@ -783,38 +781,32 @@ class JsonManager(ImportManager):
         print(list(extract_dict_values(item, filtr)))
         [("description", "Commentaire"), ("y", 1.0), ("x", -1.0)]
         """
-        for k in filtr:
-            if k not in item:
-                continue
-            if not isinstance(filtr[k], dict):
-                yield filtr[k], item[k]
-                continue
-            for key, value in self.extract_dict_values(item[k], filtr[k]):
-                yield key, value
-
-    def get(self):
-        """
-        Get data from a json simple source
+        top = False
+        if not base_key:
+            top = True
+        if not isinstance(filtr, dict):
+            if filtr in item:
+                yield base_key, item[filtr]
+        else:
+            for k in filtr:
+                if top:
+                    base_key = k
+                if not isinstance(filtr[k], dict):
+                    if k in item:
+                        yield base_key, item[k]
+                    continue
+                item_k = list(filtr[k].keys())[0]
+                if item_k not in item:
+                    continue
+                for key, value in self.extract_dict_values(
+                        item[item_k], filtr[k][item_k], base_key=base_key):
+                    yield key, value
 
-        Return a tuple with:
-         - number of new item ;
-         - number of item updated ;
-         - error detail on error
-        """
+    def _parse_json(self, values, default_filtr=None):
+        if not default_filtr:
+            default_filtr = {}
         from chimere.models import Marker
         new_item, updated_item, msg = 0, 0, ''
-        source, msg = self.get_source_file(['.json'])
-        if msg:
-            return (0, 0, msg)
-
-        vals = source.read().decode("utf-8").replace('\n', ' ')
-        try:
-            values = json.JSONDecoder(
-                object_pairs_hook=collections.OrderedDict).decode(vals)
-        except ValueError as e:
-            return (new_item, updated_item,
-                    _("JSON file is not well formed: ") + str(e))
-
         filtr = self.importer_instance.filtr
         # a left part before "{" indicate keys to be used to access to the
         # event list - separated by ";"
@@ -830,13 +822,16 @@ class JsonManager(ImportManager):
                 values = values[key]
 
         # configuration in filtr
-        try:
-            filtr = json.JSONDecoder().decode(filtr)
-        except ValueError:
-            return (
-                new_item, updated_item,
-                _("Bad configuration: filter field must be a valid "
-                  "JSON string"))
+        dct = deepcopy(default_filtr)
+        if filtr:
+            try:
+                dct.update(json.JSONDecoder().decode(filtr))
+            except ValueError:
+                return (
+                    new_item, updated_item,
+                    _("Bad configuration: filter field must be a valid "
+                      "JSON string"))
+        filtr = dct
 
         # check that mandatory fields are available
         vls = []
@@ -851,7 +846,7 @@ class JsonManager(ImportManager):
                     vls.append(val)
             cvalues = new_values
 
-        for k in ('name', 'id', 'description'):
+        for k in ('name', 'id',):
             if k not in vls:
                 return (
                     new_item, updated_item,
@@ -865,8 +860,6 @@ class JsonManager(ImportManager):
             default_dct['name'] = filtr.pop('prefix_name')
         if 'prefix_description' in filtr:
             default_dct['description'] = filtr.pop('prefix_description')
-        if self.importer_instance.default_localisation:
-            default_dct['point'] = self.importer_instance.default_localisation
 
         for item in values:
             dct = default_dct.copy()
@@ -888,6 +881,13 @@ class JsonManager(ImportManager):
                         dct[key] += " "
                 dct[key] += str(value) if value else ""
 
+            geom_type = 'point'
+            if "geom_type" in dct:
+                geom_type = dct.pop("geom_type").lower()
+
+            if self.importer_instance.default_localisation:
+                dct[geom_type] = self.importer_instance.default_localisation
+
             if 'point' in dct and isinstance(dct['point'], str):
                 x, y = dct['point'].split(",")
                 dct['point'] = 'SRID=4326;POINT(%s %s)' % (x, y)
@@ -899,7 +899,16 @@ class JsonManager(ImportManager):
                     and 'y' in dct and dct['y']:
                 dct['point'] = 'SRID=4326;POINT(%s %s)' % (dct['x'],
                                                            dct['y'])
-            if not dct['point']:
+
+            if 'coordinates' in dct:
+                coordinates = dct.pop('coordinates')
+                if geom_type == 'point':
+                    coordinates = coordinates[1:-1].split(",")
+                    dct[geom_type] = 'SRID=4326;POINT({} {})'.format(
+                        coordinates[0].strip(),
+                        coordinates[1].strip(),
+                    )
+            if geom_type not in dct or not dct[geom_type]:
                 continue
             # manage prefixes and suffixes
             for k in filtr:
@@ -918,7 +927,7 @@ class JsonManager(ImportManager):
 
             cls = Marker
             pl_id = (dct.pop('id') if 'id' in dct else dct['name']) \
-                + "-" + str(self.importer_instance.pk)
+                    + "-" + str(self.importer_instance.pk)
 
             it, updated, created = self.create_or_update_item(cls, dct, pl_id)
             if updated:
@@ -927,18 +936,63 @@ class JsonManager(ImportManager):
                 new_item += 1
         return new_item, updated_item, msg
 
+    def get(self):
+        """
+        Get data from a json simple source
+
+        Return a tuple with:
+         - number of new item ;
+         - number of item updated ;
+         - error detail on error
+        """
+        source, msg = self.get_source_file(['.json'])
+        if msg:
+            return (0, 0, msg)
+
+        vals = source.read().decode("utf-8").replace('\n', ' ')
+        try:
+            values = json.JSONDecoder(
+                object_pairs_hook=collections.OrderedDict).decode(vals)
+        except ValueError as e:
+            return (0, 0,
+                    _("JSON file is not well formed: ") + str(e))
+        return self._parse_json(values)
+
+
 RE_HOOK = re.compile('\[([^\]]*)\]')
 
 # TODO: manage deleted item from OSM
 
 
-class OSMManager(ImportManager):
+class OSMManager(JsonManager):
     """
     OSM importer/exporter
-    The source url is a path to an OSM file or a XAPI url
-    The filtr argument is XAPI args or empty if it is an OSM file.
+    The source url is a path to an OSM file or a Overpass url
+    The filtr argument is Overpass args or empty if it is an OSM file.
     """
-    default_source = settings.CHIMERE_XAPI_URL
+    default_source = settings.CHIMERE_OVERPASS_URL
+
+    def parse_overpass(self):
+        api = overpass.API(endpoint=self.default_source,
+                           timeout=600)
+        response = api.get(self.importer_instance.source)
+
+        if not response or "features" not in response:
+            return (
+                0, 0,
+                str(
+                    _("Bad response from OSM server: {}. Check your overpass "
+                      "string and that overpass servre is up.")
+                ).format(self.default_source)
+            )
+
+        default_filtr = {
+            'coordinates': {'geometry': 'coordinates'},
+            'id': 'id',
+            'name': {'properties': 'name'},
+            'geom_type': {'geometry': 'type'}
+        }
+        return self._parse_json(response['features'], default_filtr)
 
     def get(self):
         """
@@ -949,6 +1003,10 @@ class OSMManager(ImportManager):
         - updated items;
         - error detail on error.
         """
+
+        is_file = True if self.importer_instance.source_file else False
+        if not is_file:
+            return self.parse_overpass()
         source, msg = self.get_source_file(
             ['.osm'], extra_url=self.importer_instance.filtr)
         if not source: