Admin: import geojson files for towns

1 files changed, 236 insertions, 2 deletions

diff --git a/ishtar_common/admin.py b/ishtar_common/admin.py
index f64f04ad5..9ad8f304e 100644
--- a/ishtar_common/admin.py
+++ b/ishtar_common/admin.py
@@ -18,7 +18,12 @@
 # See the file COPYING for details.
 
 import csv
+import json
+import os
+import shutil
+import tempfile
 import urllib
+import zipfile
 
 from ajax_select import make_ajax_form
 from ajax_select.fields import AutoCompleteSelectField, \
@@ -34,6 +39,9 @@ from django.contrib.contenttypes.models import ContentType
 from django.contrib.sites.admin import SiteAdmin
 from django.contrib.sites.models import Site
 from django.contrib.gis.forms import PointField, OSMWidget, MultiPolygonField
+from django.contrib.gis.geos import GEOSGeometry, MultiPolygon
+from django.contrib.gis.gdal.error import GDALException
+from django.contrib.gis.geos.error import GEOSException
 from django.core.cache import cache
 from django.core.urlresolvers import reverse
 from django.db.models.fields import BooleanField, IntegerField, FloatField, \
@@ -385,7 +393,231 @@ class ImportActionAdmin(admin.ModelAdmin):
                 return HttpResponseRedirect(url)
         if not form:
             form = ImportGenericForm()
-        return render(request, 'admin/import_from_csv.html', {'csv_form': form})
+        return render(
+            request, 'admin/import_from_file.html',
+            {'file_form': form, 'current_action': 'import_generic'}
+        )
+
+
+class ImportGeoJsonForm(forms.Form):
+    json_file = forms.FileField(
+        _(u"Geojson file"),
+        help_text=_(u"Only unicode encoding is managed - convert your"
+                    u" file first. The file must be a geojson file or a zip "
+                    u"containing a geojson file.")
+    )
+    numero_insee_prefix = forms.CharField(
+        label=_(u"Prefix for numero INSEE"), max_length=20, required=False)
+    numero_insee_name = forms.CharField(
+        label=_(u"Field name for numero INSEE"), max_length=200,
+        initial='code_insee')
+    name_name = forms.CharField(
+        label=_(u"Field name for name"), max_length=200, initial='nom')
+    UNIT_CHOICES = (('1', _(u"m2")), ('1000', _(u"km2")))
+    surface_unit = forms.ChoiceField(
+        label=_(u"Surface unit"), choices=UNIT_CHOICES)
+    surface_name = forms.CharField(
+        label=_(u"Field name for surface"), max_length=200, required=False)
+    year_name = forms.CharField(
+        label=_(u"Field name for year"), max_length=200, required=False
+    )
+
+
+class ImportGEOJSONActionAdmin(object):
+    def get_urls(self):
+        urls = super(ImportGEOJSONActionAdmin, self).get_urls()
+        my_urls = [
+            url(r'^import-from-geojson/$', self.import_geojson),
+        ]
+        return my_urls + urls
+
+    def geojson_values(self, request, idx, feature, keys, trace_error=True):
+        for key in ("geometry", "properties"):
+            if key in feature:
+                continue
+            if trace_error:
+                error = unicode(
+                    _(u"\"{}\" not found in feature {}")
+                ).format(key, idx)
+                self.message_user(request, error, level=messages.ERROR)
+            return False
+
+        values = {}
+        for key in keys:
+            if not key.endswith('_name'):
+                continue
+            if not keys[key]:
+                values[key[:-len("_name")]] = None
+                continue
+            if keys[key] not in feature['properties']:
+                if trace_error:
+                    error = unicode(
+                        _(u"\"{}\" not found in properties of feature {}")
+                    ).format(keys[key], idx)
+                    self.message_user(request, error,
+                                      level=messages.ERROR)
+                return False
+            value = feature['properties'][keys[key]]
+            if key == 'numero_insee_name' and keys['insee_prefix']:
+                value = keys['insee_prefix'] + value
+            values[key[:-len("_name")]] = value
+        try:
+            geom = GEOSGeometry(json.dumps(feature['geometry']))
+        except (GEOSException, GDALException):
+            if trace_error:
+                error = unicode(
+                    _(u"Bad geometry for feature {}")
+                ).format(idx)
+                self.message_user(request, error,
+                                  level=messages.ERROR)
+            return False
+        if geom.geom_type == 'Point':
+            values['center'] = geom
+        elif geom.geom_type == 'MultiPolygon':
+            values['limit'] = geom
+        elif geom.geom_type == 'Polygon':
+            values['limit'] = MultiPolygon(geom)
+        else:
+            if trace_error:
+                error = unicode(
+                    _(u"Geometry {} not managed for towns - feature {}")
+                ).format(geom.geom_type, idx)
+                self.message_user(request, error,
+                                  level=messages.ERROR)
+            return False
+        if keys['surface_unit'] and values['surface']:
+            try:
+                values['surface'] = keys['surface_unit'] * int(
+                    values['surface'])
+            except ValueError:
+                if trace_error:
+                    error = unicode(
+                        _(u"Bad value for surface: {} - feature {}")
+                    ).format(values['surface'], idx)
+                    self.message_user(request, error,
+                                      level=messages.ERROR)
+                return False
+
+        return values
+
+    def import_geojson_clean(self, tempdir):
+        if not tempdir:
+            return
+        shutil.rmtree(tempdir)
+
+    def import_geojson_error(self, request, error, base_dct, tempdir=None):
+        self.import_geojson_clean(tempdir)
+        self.message_user(request, error, level=messages.ERROR)
+        return render(
+            request, 'admin/import_from_file.html',
+            base_dct)
+
+    def import_geojson(self, request):
+        form = None
+
+        if 'apply' in request.POST:
+            form = ImportGeoJsonForm(request.POST, request.FILES)
+            if form.is_valid():
+                json_file_obj = request.FILES['json_file']
+
+                base_dct = {'file_form': form,
+                            'current_action': 'import_geojson'}
+
+                tempdir = tempfile.mkdtemp()
+                tmpfilename = tempdir + os.sep + "dest_file"
+                tmpfile = open(tmpfilename, 'wb+')
+                for chunk in json_file_obj.chunks():
+                    tmpfile.write(chunk)
+                tmpfile.close()
+
+                json_filename = None
+                if zipfile.is_zipfile(tmpfilename):
+                    zfile = zipfile.ZipFile(tmpfilename)
+                    for zmember in zfile.namelist():
+                        if os.sep in zmember or u".." in zmember:
+                            continue
+                        if zmember.endswith("json"):
+                            zfile.extract(zmember, tempdir)
+                            json_filename = tempdir + os.sep + zmember
+                            break
+                    if not json_filename:
+                        error = _(u"No json file found in zipfile")
+                        return self.import_geojson_error(request, error,
+                                                         base_dct, tempdir)
+                else:
+                    json_filename = tmpfilename
+
+                keys = {
+                    "numero_insee_name": request.POST.get('numero_insee_name'),
+                    "name_name": request.POST.get('name_name'),
+                    "surface_name": request.POST.get('surface_name', "") or "",
+                    "year_name": request.POST.get('year_name', "") or "",
+                    "insee_prefix": request.POST.get('numero_insee_prefix',
+                                                     None) or '',
+                    "surface_unit": int(request.POST.get('surface_unit'))
+                }
+                created = 0
+                updated = 0
+                with open(json_filename) as json_file_obj:
+                    json_file = json_file_obj.read()
+                    try:
+                        dct = json.loads(json_file)
+                        assert 'features' in dct
+                        assert dct['features']
+                    except (ValueError, AssertionError):
+                        error = _(u"Bad geojson file")
+                        return self.import_geojson_error(
+                            request, error, base_dct, tempdir)
+
+                    error_count = 0
+                    for idx, feat in enumerate(dct['features']):
+                        trace_error = True
+                        if error_count == 6:
+                            self.message_user(request, _(u"Too many errors..."),
+                                              level=messages.ERROR)
+                        if error_count > 5:
+                            trace_error = False
+                        values = self.geojson_values(
+                            request, idx + 1, feat, keys, trace_error
+                        )
+                        if not values:
+                            error_count += 1
+                            continue
+                        num_insee = values.pop('numero_insee')
+                        year = values.pop('year') or None
+                        t, c = models.Town.objects.get_or_create(
+                            numero_insee=num_insee, year=year,
+                            defaults=values)
+                        if c:
+                            created += 1
+                        else:
+                            modified = False
+                            for k in values:
+                                if values[k] != getattr(t, k):
+                                    setattr(t, k, values[k])
+                                    modified = True
+                            if modified:
+                                updated += 1
+                                t.save()
+                    if created:
+                        self.message_user(
+                            request,
+                            unicode(_(u"%d item(s) created.")) % created)
+                    if updated:
+                        self.message_user(
+                            request,
+                            unicode(_(u"%d item(s) updated.")) % updated)
+                    self.import_geojson_clean(tempdir)
+                    url = reverse(
+                        'admin:%s_%s_changelist' % (
+                            self.model._meta.app_label, self.model._meta.model_name)
+                    )
+                    return HttpResponseRedirect(url)
+        if not form:
+            form = ImportGeoJsonForm()
+        return render(
+            request, 'admin/import_from_file.html',
+            {'file_form': form, 'current_action': 'import_geojson'})
 
 
 class AdminRelatedTownForm(forms.ModelForm):
@@ -417,7 +649,9 @@ class TownParentInline(admin.TabularInline):
     extra = 1
 
 
-class TownAdmin(ImportActionAdmin):
+class TownAdmin(ImportGEOJSONActionAdmin, ImportActionAdmin):
+    change_list_template = "admin/town_change_list.html"
+
     model = models.Town
     list_display = ['name', 'year']
     search_fields = ['name']