Use overpass API for OSM imports

8 files changed, 164 insertions, 118 deletions

diff --git a/chimere/forms.py b/chimere/forms.py
index 1af454b..1178c81 100644
--- a/chimere/forms.py
+++ b/chimere/forms.py
@@ -204,7 +204,6 @@ class NewsAdminForm(forms.ModelForm):
 
 
 class ImporterAdminForm(forms.ModelForm):
-    filtr = forms.CharField(widget=ImportFiltrWidget, required=False)
     importer_type = forms.ChoiceField(
         widget=ImporterChoicesWidget,
         choices=[('', '--')] + list(IMPORTER_CHOICES))
@@ -224,16 +223,6 @@ class ImporterAdminForm(forms.ModelForm):
         Verify that only one type of source is provided
         Verify that shapefiles are zipped
         """
-        if self.cleaned_data.get('importer_type') == 'OSM' and \
-           not self.cleaned_data.get('filtr'):
-            raise forms.ValidationError(
-                _("For OSM import you must be provide a filter. Select an "
-                  "area and node/way filter."))
-        if self.cleaned_data.get('importer_type') == 'OSM' and \
-           not RE_XAPI.match(self.cleaned_data.get('filtr')):
-            raise forms.ValidationError(
-                _("For OSM import you must be provide a filter. Select an "
-                  "area and node/way filter."))
         if self.cleaned_data.get('importer_type') == 'SHP' and \
            not self.cleaned_data.get('zipped'):
             raise forms.ValidationError(_("Shapefiles must be provided in a "
@@ -244,8 +233,7 @@ class ImporterAdminForm(forms.ModelForm):
             raise forms.ValidationError(_("You have to set \"source\" or "
                                           "\"source file\" but not both."))
         if not self.cleaned_data.get('source') and \
-           not self.cleaned_data.get('source_file') and \
-           self.cleaned_data.get('importer_type') != 'OSM':
+           not self.cleaned_data.get('source_file'):
             raise forms.ValidationError(_("You have to set \"source\" or "
                                           "\"source file\"."))
         return self.cleaned_data
diff --git a/chimere/models.py b/chimere/models.py
index ccbe4ef..5fc946c 100644
--- a/chimere/models.py
+++ b/chimere/models.py
@@ -473,7 +473,6 @@ class Importer(models.Model):
     """
     importer_type = models.CharField(_("Importer type"), max_length=4,
                                      choices=IMPORTER_CHOICES)
-    filtr = models.TextField(_("Filter"), blank=True, null=True)
     source = models.CharField(_("Web address"), max_length=200,
                               blank=True, null=True,
                               help_text=_("Don't forget the trailing slash"))
@@ -481,6 +480,7 @@ class Importer(models.Model):
         _("Source file"), upload_to='import_files', blank=True, null=True)
     source_file_alt = models.FileField(
         _("Alt source file"), upload_to='import_files', blank=True, null=True)
+    filtr = models.TextField(_("Filter"), blank=True, null=True, help_text=" ")
     default_name = models.CharField(_("Name by default"), max_length=200,
                                     blank=True, null=True)
     srid = models.IntegerField(_("SRID"), blank=True, null=True)
diff --git a/chimere/static/chimere/css/forms.css b/chimere/static/chimere/css/forms.css
index 0a22e74..a931092 100644
--- a/chimere/static/chimere/css/forms.css
+++ b/chimere/static/chimere/css/forms.css
@@ -155,8 +155,7 @@ div.bottomform{
 }
 
 .form-row.field-route,
-.form-row.field-point,
-.form-row.field-filtr.field-map{
+.form-row.field-point{
     float:right;
     width:50%;
 }
diff --git a/chimere/static/chimere/js/importer_interface.js b/chimere/static/chimere/js/importer_interface.js
index 9cc1f3c..a423411 100644
--- a/chimere/static/chimere/js/importer_interface.js
+++ b/chimere/static/chimere/js/importer_interface.js
@@ -1,8 +1,26 @@
+var init_widget_list = [];
+
 django.jQuery(function($) {
+    var labels = {
+        OSM: {
+            "source": "Expression Overpass :"
+        }
+    };
+    var helps = {
+        OSM: {
+            "source": "Une seule expression est permise. Par exemple : node[\"railway\"=\"station\"](41.55,1,42,1.2)",
+            "filtr": "Les propriétés OSM peuvent être associées à des attributs" +
+                     "en définissant le dictionnaire adéquat. (cf. documentation)"
+        }
+    };
+    var default_label = {};
+    var default_help = {};
     var importer_form_filter = {
-        OSM:new Array('field-filtr', 'field-default_name', 'field-categories',
-                      'field-source', 'field-overwrite',
-                      'field-automatic_update', 'field-default_status'),
+        OSM:new Array('field-source', 'field-source_file', 'field-default_name',
+                      'field-filtr', 'field-zipped', 'field-origin',
+                      'field-license', 'field-categories', 'field-overwrite',
+                      'field-get_description', 'field-automatic_update',
+                      'field-default_status', 'field-default_localisation'),
         KML:new Array('field-source', 'field-source_file', 'field-default_name',
                       'field-filtr', 'field-zipped', 'field-origin',
                       'field-license', 'field-categories', 'field-overwrite',
@@ -61,22 +79,9 @@ django.jQuery(function($) {
         } else {
             $('.help-kml').hide();
         }
-        if (importer_val == 'OSM'){
-            $('.form-row.field-filtr').addClass('field-map');
-            $('#map_edit_area').show();
-            if(!$('#id_source').val()){
-                $('#id_source').val(default_xapi);
-            }
-            $('#id_filtr').attr('readonly', true);
-            $('.help-osm').show();
-            $('.input-osm').show();
-            if (!osm_map_initialized){
-                init_map_form();
-                osm_map_initialized = true;
-            }
-        }
-        else if (importer_val == 'XSLT' || importer_val == 'XXLT'
-                 || importer_val == 'JSON' || importer_val == 'ICAL'){
+        if (importer_val == 'XSLT' || importer_val == 'XXLT'
+                 || importer_val == 'JSON' || importer_val == 'ICAL' ||
+                 importer_val == 'OSM'){
             $('#importerkeycategories_set-group').show();
             $('#key_categories-group').show();
             $('#importerkeycategories_set-group .form-row').show();
@@ -84,8 +89,6 @@ django.jQuery(function($) {
             $('.form-row.field-filtr').addClass('field-map');
             $('#map_edit').show();
             $('#map_edit_area').hide();
-            $('.help-osm').hide();
-            $('.input-osm').hide();
             if (!edit_map_initialized){
                 init_map_edit();
                 edit_map_initialized = true;
@@ -95,9 +98,28 @@ django.jQuery(function($) {
             $('#id_filtr').attr('readonly', false);
             $('#map_edit_area').hide();
             $('#map_edit').hide();
-            $('.help-osm').hide();
-            $('.input-osm').hide();
-            if($('#id_source').val() == default_xapi) $('#id_source').val('');
+        }
+        for (key in default_label){  // restore default label
+            $(".field-" + key + " label").html(default_label[key]);
+        }
+        if (importer_val in labels){
+            for (key in labels[importer_val]){
+                if (!(key in default_label)){
+                    default_label[key] = $(".field-" + key + " label").html();
+                }
+                $(".field-" + key + " label").html(labels[importer_val][key]);
+            }
+        }
+        for (key in default_help){  // restore default help
+            $(".field-" + key + " .help").html(default_help[key]);
+        }
+        if (importer_val in helps){
+            for (key in helps[importer_val]){
+                if (!(key in default_help)){
+                    default_help[key] = $(".field-" + key + " .help").html();
+                }
+                $(".field-" + key + " .help").html(helps[importer_val][key]);
+            }
         }
         refresh_default_desc();
     }
diff --git a/chimere/utils.py b/chimere/utils.py
index 707c6bb..a5772d7 100644
--- a/chimere/utils.py
+++ b/chimere/utils.py
@@ -39,6 +39,7 @@ import zipfile
 
 from osgeo import ogr, osr
 from osmapi import OsmApi
+import overpass
 from lxml import etree
 
 from django.conf import settings
@@ -140,9 +141,6 @@ class ImportManager(object):
             values.update({
                 'import_source': self.importer_instance.source})
             values['status'] = self.importer_instance.default_status
-            item = cls.objects.create(**values)
-            item.modified_since_import = False
-            item.save()
             try:
                 item = cls.objects.create(**values)
                 item.modified_since_import = False
@@ -766,7 +764,7 @@ class JsonManager(ImportManager):
     This manager only gets and do not produce Json feed
     """
 
-    def extract_dict_values(self, item, filtr):
+    def extract_dict_values(self, item, filtr, base_key=None):
         """
         Extract values from a dict.
 
@@ -783,38 +781,32 @@ class JsonManager(ImportManager):
         print(list(extract_dict_values(item, filtr)))
         [("description", "Commentaire"), ("y", 1.0), ("x", -1.0)]
         """
-        for k in filtr:
-            if k not in item:
-                continue
-            if not isinstance(filtr[k], dict):
-                yield filtr[k], item[k]
-                continue
-            for key, value in self.extract_dict_values(item[k], filtr[k]):
-                yield key, value
-
-    def get(self):
-        """
-        Get data from a json simple source
+        top = False
+        if not base_key:
+            top = True
+        if not isinstance(filtr, dict):
+            if filtr in item:
+                yield base_key, item[filtr]
+        else:
+            for k in filtr:
+                if top:
+                    base_key = k
+                if not isinstance(filtr[k], dict):
+                    if k in item:
+                        yield base_key, item[k]
+                    continue
+                item_k = list(filtr[k].keys())[0]
+                if item_k not in item:
+                    continue
+                for key, value in self.extract_dict_values(
+                        item[item_k], filtr[k][item_k], base_key=base_key):
+                    yield key, value
 
-        Return a tuple with:
-         - number of new item ;
-         - number of item updated ;
-         - error detail on error
-        """
+    def _parse_json(self, values, default_filtr=None):
+        if not default_filtr:
+            default_filtr = {}
         from chimere.models import Marker
         new_item, updated_item, msg = 0, 0, ''
-        source, msg = self.get_source_file(['.json'])
-        if msg:
-            return (0, 0, msg)
-
-        vals = source.read().decode("utf-8").replace('\n', ' ')
-        try:
-            values = json.JSONDecoder(
-                object_pairs_hook=collections.OrderedDict).decode(vals)
-        except ValueError as e:
-            return (new_item, updated_item,
-                    _("JSON file is not well formed: ") + str(e))
-
         filtr = self.importer_instance.filtr
         # a left part before "{" indicate keys to be used to access to the
         # event list - separated by ";"
@@ -830,13 +822,16 @@ class JsonManager(ImportManager):
                 values = values[key]
 
         # configuration in filtr
-        try:
-            filtr = json.JSONDecoder().decode(filtr)
-        except ValueError:
-            return (
-                new_item, updated_item,
-                _("Bad configuration: filter field must be a valid "
-                  "JSON string"))
+        dct = deepcopy(default_filtr)
+        if filtr:
+            try:
+                dct.update(json.JSONDecoder().decode(filtr))
+            except ValueError:
+                return (
+                    new_item, updated_item,
+                    _("Bad configuration: filter field must be a valid "
+                      "JSON string"))
+        filtr = dct
 
         # check that mandatory fields are available
         vls = []
@@ -851,7 +846,7 @@ class JsonManager(ImportManager):
                     vls.append(val)
             cvalues = new_values
 
-        for k in ('name', 'id', 'description'):
+        for k in ('name', 'id',):
             if k not in vls:
                 return (
                     new_item, updated_item,
@@ -865,8 +860,6 @@ class JsonManager(ImportManager):
             default_dct['name'] = filtr.pop('prefix_name')
         if 'prefix_description' in filtr:
             default_dct['description'] = filtr.pop('prefix_description')
-        if self.importer_instance.default_localisation:
-            default_dct['point'] = self.importer_instance.default_localisation
 
         for item in values:
             dct = default_dct.copy()
@@ -888,6 +881,13 @@ class JsonManager(ImportManager):
                         dct[key] += " "
                 dct[key] += str(value) if value else ""
 
+            geom_type = 'point'
+            if "geom_type" in dct:
+                geom_type = dct.pop("geom_type").lower()
+
+            if self.importer_instance.default_localisation:
+                dct[geom_type] = self.importer_instance.default_localisation
+
             if 'point' in dct and isinstance(dct['point'], str):
                 x, y = dct['point'].split(",")
                 dct['point'] = 'SRID=4326;POINT(%s %s)' % (x, y)
@@ -899,7 +899,16 @@ class JsonManager(ImportManager):
                     and 'y' in dct and dct['y']:
                 dct['point'] = 'SRID=4326;POINT(%s %s)' % (dct['x'],
                                                            dct['y'])
-            if not dct['point']:
+
+            if 'coordinates' in dct:
+                coordinates = dct.pop('coordinates')
+                if geom_type == 'point':
+                    coordinates = coordinates[1:-1].split(",")
+                    dct[geom_type] = 'SRID=4326;POINT({} {})'.format(
+                        coordinates[0].strip(),
+                        coordinates[1].strip(),
+                    )
+            if geom_type not in dct or not dct[geom_type]:
                 continue
             # manage prefixes and suffixes
             for k in filtr:
@@ -918,7 +927,7 @@ class JsonManager(ImportManager):
 
             cls = Marker
             pl_id = (dct.pop('id') if 'id' in dct else dct['name']) \
-                + "-" + str(self.importer_instance.pk)
+                    + "-" + str(self.importer_instance.pk)
 
             it, updated, created = self.create_or_update_item(cls, dct, pl_id)
             if updated:
@@ -927,18 +936,63 @@ class JsonManager(ImportManager):
                 new_item += 1
         return new_item, updated_item, msg
 
+    def get(self):
+        """
+        Get data from a json simple source
+
+        Return a tuple with:
+         - number of new item ;
+         - number of item updated ;
+         - error detail on error
+        """
+        source, msg = self.get_source_file(['.json'])
+        if msg:
+            return (0, 0, msg)
+
+        vals = source.read().decode("utf-8").replace('\n', ' ')
+        try:
+            values = json.JSONDecoder(
+                object_pairs_hook=collections.OrderedDict).decode(vals)
+        except ValueError as e:
+            return (0, 0,
+                    _("JSON file is not well formed: ") + str(e))
+        return self._parse_json(values)
+
+
 RE_HOOK = re.compile('\[([^\]]*)\]')
 
 # TODO: manage deleted item from OSM
 
 
-class OSMManager(ImportManager):
+class OSMManager(JsonManager):
     """
     OSM importer/exporter
-    The source url is a path to an OSM file or a XAPI url
-    The filtr argument is XAPI args or empty if it is an OSM file.
+    The source url is a path to an OSM file or a Overpass url
+    The filtr argument is Overpass args or empty if it is an OSM file.
     """
-    default_source = settings.CHIMERE_XAPI_URL
+    default_source = settings.CHIMERE_OVERPASS_URL
+
+    def parse_overpass(self):
+        api = overpass.API(endpoint=self.default_source,
+                           timeout=600)
+        response = api.get(self.importer_instance.source)
+
+        if not response or "features" not in response:
+            return (
+                0, 0,
+                str(
+                    _("Bad response from OSM server: {}. Check your overpass "
+                      "string and that overpass servre is up.")
+                ).format(self.default_source)
+            )
+
+        default_filtr = {
+            'coordinates': {'geometry': 'coordinates'},
+            'id': 'id',
+            'name': {'properties': 'name'},
+            'geom_type': {'geometry': 'type'}
+        }
+        return self._parse_json(response['features'], default_filtr)
 
     def get(self):
         """
@@ -949,6 +1003,10 @@ class OSMManager(ImportManager):
         - updated items;
         - error detail on error.
         """
+
+        is_file = True if self.importer_instance.source_file else False
+        if not is_file:
+            return self.parse_overpass()
         source, msg = self.get_source_file(
             ['.osm'], extra_url=self.importer_instance.filtr)
         if not source:
diff --git a/chimere/widgets.py b/chimere/widgets.py
index 1ab5e44..346e3cf 100644
--- a/chimere/widgets.py
+++ b/chimere/widgets.py
@@ -557,9 +557,6 @@ class AreaWidget(forms.TextInput):
             " value='%f'/>\n" % (
                 upper_left_lat, upper_left_lon, lower_right_lat,
                 lower_right_lon)
-        help_msg = _("Click to begin selecting area on the map and click again "
-                     "to close the rectangle. To modify, move the nodes of the rectangle.")
-        tpl += "<p class='help-osm'>%s</p>\n" % help_msg
         tpl += "<script type='text/javascript'>\n"
         tpl += "function init_map_form (){\ninit('map_edit_area');\n"
         if value:
@@ -663,29 +660,14 @@ class ImportFiltrWidget(AreaWidget):
         """
         tpl = super(ImportFiltrWidget, self).render(name, value, attrs,
                                                     initialized=False)
-        tpl += "</div><hr class='spacer'/>"
-        vals = {'lbl': _("Type:"), 'name': name, 'node': _("Node"),
-                'way': _("Way")}
+        vals = {
+            'lbl': _("Type:"), 'name': name, 'node': _("Node"),
+            'way': _("Way")
+        }
         vals['way_selected'] = ' checked="checked"'\
             if self.xapi_type == 'way' else ''
         vals['node_selected'] = ' checked="checked"'\
             if self.xapi_type == 'node' else ''
-        tpl += "<div class='input-osm'><label>%(lbl)s</label>"\
-            "<input type='radio' name='id_%(name)s_type' "\
-            "id='id_%(name)s_node' value='node'%(node_selected)s/> "\
-            "<label for='id_%(name)s_node'>"\
-            "%(node)s</label> <input type='radio' name='id_%(name)s_type' "\
-            "id='id_%(name)s_way' value='way'%(way_selected)s/> <label "\
-            "for='id_%(name)s_way'>%(way)s</label></div>" % vals
-        help_msg = _(
-            "Enter an OSM \"tag=value\" string such as "
-            "\"amenity=pub\". A list of common tag is available "
-            "<a href='https://wiki.openstreetmap.org/wiki/Map_Features' "
-            " target='_blank'>here</a>.")
-        tpl += "<p class='help-osm'>%s</p>\n" % help_msg
-        tpl += "<div class='input-osm'><label for='id_%s_tag'>%s</label>"\
-               "<input type='text' id='id_%s_tag' value=\"%s\"/></div>" % (
-                   name, _("Tag:"), name, self.xapi_tag)
         tpl += "<script type='text/javascript'>\n"
         tpl += "var default_xapi='%s';" % settings.CHIMERE_XAPI_URL
         tpl += 'var msg_missing_area = "%s";' % \
@@ -695,18 +677,13 @@ class ImportFiltrWidget(AreaWidget):
         tpl += 'var msg_missing_filtr = "%s";' % \
             _("You have to insert a filter tag.")
         tpl += "</script>\n"
-        help_msg = _("If you change the above form don't forget to refresh "
-                     "before submit!")
-        tpl += "<p class='help-osm errornote'>%s</p>\n" % help_msg
         help_msg = _("You can put a Folder name of the KML file to filter on "
                      "it.")
         tpl += "<p class='help-kml'>%s</p>\n" % help_msg
         if not value:
             value = ''
-        tpl += "<div><textarea id='id_%s' name='id_%s' "\
-               ">%s</textarea> <input type='button' id='id_refresh_%s' "\
-               "value='%s' class='input-osm'/>" % (name, name, value, name,
-                                                   _("Refresh"))
+        tpl += "<textarea id='id_%s' name='id_%s'>%s</textarea>" % (
+            name, name, value)
         return mark_safe(tpl)
 
     def value_from_datadict(self, data, files, name):
diff --git a/requirements.txt b/requirements.txt
index fc2cc4c..3d6e7ba 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,3 +11,4 @@ py3exiv2==0.2.1
 gdal>=2.1.0,<2.1.99
 osmapi==0.6.2
 PyTidyLib==0.3.1
+overpass==0.6.0
diff --git a/settings.py b/settings.py
index b674977..878d1cd 100644
--- a/settings.py
+++ b/settings.py
@@ -74,6 +74,7 @@ CHIMERE_DEFAULT_MAP_LAYER = """new ol.layer.Tile({
 })"""
 
 CHIMERE_XAPI_URL = "http://www.overpass-api.de/api/xapi_meta?"
+CHIMERE_OVERPASS_URL = "https://lz4.overpass-api.de/api/interpreter"
 CHIMERE_OSM_API_URL = 'api06.dev.openstreetmap.org'  # test URL
 CHIMERE_OSM_USER = 'test'
 CHIMERE_OSM_PASSWORD = 'test'