diff options
| -rw-r--r-- | ishtar_common/admin.py | 238 | ||||
| -rw-r--r-- | ishtar_common/templates/admin/import_from_file.html (renamed from ishtar_common/templates/admin/import_from_csv.html) | 4 | ||||
| -rw-r--r-- | ishtar_common/templates/admin/town_change_list.html | 24 |
3 files changed, 262 insertions, 4 deletions
diff --git a/ishtar_common/admin.py b/ishtar_common/admin.py index f64f04ad5..9ad8f304e 100644 --- a/ishtar_common/admin.py +++ b/ishtar_common/admin.py @@ -18,7 +18,12 @@ # See the file COPYING for details. import csv +import json +import os +import shutil +import tempfile import urllib +import zipfile from ajax_select import make_ajax_form from ajax_select.fields import AutoCompleteSelectField, \ @@ -34,6 +39,9 @@ from django.contrib.contenttypes.models import ContentType from django.contrib.sites.admin import SiteAdmin from django.contrib.sites.models import Site from django.contrib.gis.forms import PointField, OSMWidget, MultiPolygonField +from django.contrib.gis.geos import GEOSGeometry, MultiPolygon +from django.contrib.gis.gdal.error import GDALException +from django.contrib.gis.geos.error import GEOSException from django.core.cache import cache from django.core.urlresolvers import reverse from django.db.models.fields import BooleanField, IntegerField, FloatField, \ @@ -385,7 +393,231 @@ class ImportActionAdmin(admin.ModelAdmin): return HttpResponseRedirect(url) if not form: form = ImportGenericForm() - return render(request, 'admin/import_from_csv.html', {'csv_form': form}) + return render( + request, 'admin/import_from_file.html', + {'file_form': form, 'current_action': 'import_generic'} + ) + + +class ImportGeoJsonForm(forms.Form): + json_file = forms.FileField( + _(u"Geojson file"), + help_text=_(u"Only unicode encoding is managed - convert your" + u" file first. The file must be a geojson file or a zip " + u"containing a geojson file.") + ) + numero_insee_prefix = forms.CharField( + label=_(u"Prefix for numero INSEE"), max_length=20, required=False) + numero_insee_name = forms.CharField( + label=_(u"Field name for numero INSEE"), max_length=200, + initial='code_insee') + name_name = forms.CharField( + label=_(u"Field name for name"), max_length=200, initial='nom') + UNIT_CHOICES = (('1', _(u"m2")), ('1000', _(u"km2"))) + surface_unit = forms.ChoiceField( + label=_(u"Surface unit"), choices=UNIT_CHOICES) + surface_name = forms.CharField( + label=_(u"Field name for surface"), max_length=200, required=False) + year_name = forms.CharField( + label=_(u"Field name for year"), max_length=200, required=False + ) + + +class ImportGEOJSONActionAdmin(object): + def get_urls(self): + urls = super(ImportGEOJSONActionAdmin, self).get_urls() + my_urls = [ + url(r'^import-from-geojson/$', self.import_geojson), + ] + return my_urls + urls + + def geojson_values(self, request, idx, feature, keys, trace_error=True): + for key in ("geometry", "properties"): + if key in feature: + continue + if trace_error: + error = unicode( + _(u"\"{}\" not found in feature {}") + ).format(key, idx) + self.message_user(request, error, level=messages.ERROR) + return False + + values = {} + for key in keys: + if not key.endswith('_name'): + continue + if not keys[key]: + values[key[:-len("_name")]] = None + continue + if keys[key] not in feature['properties']: + if trace_error: + error = unicode( + _(u"\"{}\" not found in properties of feature {}") + ).format(keys[key], idx) + self.message_user(request, error, + level=messages.ERROR) + return False + value = feature['properties'][keys[key]] + if key == 'numero_insee_name' and keys['insee_prefix']: + value = keys['insee_prefix'] + value + values[key[:-len("_name")]] = value + try: + geom = GEOSGeometry(json.dumps(feature['geometry'])) + except (GEOSException, GDALException): + if trace_error: + error = unicode( + _(u"Bad geometry for feature {}") + ).format(idx) + self.message_user(request, error, + level=messages.ERROR) + return False + if geom.geom_type == 'Point': + values['center'] = geom + elif geom.geom_type == 'MultiPolygon': + values['limit'] = geom + elif geom.geom_type == 'Polygon': + values['limit'] = MultiPolygon(geom) + else: + if trace_error: + error = unicode( + _(u"Geometry {} not managed for towns - feature {}") + ).format(geom.geom_type, idx) + self.message_user(request, error, + level=messages.ERROR) + return False + if keys['surface_unit'] and values['surface']: + try: + values['surface'] = keys['surface_unit'] * int( + values['surface']) + except ValueError: + if trace_error: + error = unicode( + _(u"Bad value for surface: {} - feature {}") + ).format(values['surface'], idx) + self.message_user(request, error, + level=messages.ERROR) + return False + + return values + + def import_geojson_clean(self, tempdir): + if not tempdir: + return + shutil.rmtree(tempdir) + + def import_geojson_error(self, request, error, base_dct, tempdir=None): + self.import_geojson_clean(tempdir) + self.message_user(request, error, level=messages.ERROR) + return render( + request, 'admin/import_from_file.html', + base_dct) + + def import_geojson(self, request): + form = None + + if 'apply' in request.POST: + form = ImportGeoJsonForm(request.POST, request.FILES) + if form.is_valid(): + json_file_obj = request.FILES['json_file'] + + base_dct = {'file_form': form, + 'current_action': 'import_geojson'} + + tempdir = tempfile.mkdtemp() + tmpfilename = tempdir + os.sep + "dest_file" + tmpfile = open(tmpfilename, 'wb+') + for chunk in json_file_obj.chunks(): + tmpfile.write(chunk) + tmpfile.close() + + json_filename = None + if zipfile.is_zipfile(tmpfilename): + zfile = zipfile.ZipFile(tmpfilename) + for zmember in zfile.namelist(): + if os.sep in zmember or u".." in zmember: + continue + if zmember.endswith("json"): + zfile.extract(zmember, tempdir) + json_filename = tempdir + os.sep + zmember + break + if not json_filename: + error = _(u"No json file found in zipfile") + return self.import_geojson_error(request, error, + base_dct, tempdir) + else: + json_filename = tmpfilename + + keys = { + "numero_insee_name": request.POST.get('numero_insee_name'), + "name_name": request.POST.get('name_name'), + "surface_name": request.POST.get('surface_name', "") or "", + "year_name": request.POST.get('year_name', "") or "", + "insee_prefix": request.POST.get('numero_insee_prefix', + None) or '', + "surface_unit": int(request.POST.get('surface_unit')) + } + created = 0 + updated = 0 + with open(json_filename) as json_file_obj: + json_file = json_file_obj.read() + try: + dct = json.loads(json_file) + assert 'features' in dct + assert dct['features'] + except (ValueError, AssertionError): + error = _(u"Bad geojson file") + return self.import_geojson_error( + request, error, base_dct, tempdir) + + error_count = 0 + for idx, feat in enumerate(dct['features']): + trace_error = True + if error_count == 6: + self.message_user(request, _(u"Too many errors..."), + level=messages.ERROR) + if error_count > 5: + trace_error = False + values = self.geojson_values( + request, idx + 1, feat, keys, trace_error + ) + if not values: + error_count += 1 + continue + num_insee = values.pop('numero_insee') + year = values.pop('year') or None + t, c = models.Town.objects.get_or_create( + numero_insee=num_insee, year=year, + defaults=values) + if c: + created += 1 + else: + modified = False + for k in values: + if values[k] != getattr(t, k): + setattr(t, k, values[k]) + modified = True + if modified: + updated += 1 + t.save() + if created: + self.message_user( + request, + unicode(_(u"%d item(s) created.")) % created) + if updated: + self.message_user( + request, + unicode(_(u"%d item(s) updated.")) % updated) + self.import_geojson_clean(tempdir) + url = reverse( + 'admin:%s_%s_changelist' % ( + self.model._meta.app_label, self.model._meta.model_name) + ) + return HttpResponseRedirect(url) + if not form: + form = ImportGeoJsonForm() + return render( + request, 'admin/import_from_file.html', + {'file_form': form, 'current_action': 'import_geojson'}) class AdminRelatedTownForm(forms.ModelForm): @@ -417,7 +649,9 @@ class TownParentInline(admin.TabularInline): extra = 1 -class TownAdmin(ImportActionAdmin): +class TownAdmin(ImportGEOJSONActionAdmin, ImportActionAdmin): + change_list_template = "admin/town_change_list.html" + model = models.Town list_display = ['name', 'year'] search_fields = ['name'] diff --git a/ishtar_common/templates/admin/import_from_csv.html b/ishtar_common/templates/admin/import_from_file.html index 282cbcca2..578a489d6 100644 --- a/ishtar_common/templates/admin/import_from_csv.html +++ b/ishtar_common/templates/admin/import_from_file.html @@ -5,9 +5,9 @@ <form action="." method="post" enctype="multipart/form-data"> {% csrf_token %} <table> -{{ csv_form }} +{{ file_form }} </table> -<input type="hidden" name="action" value="import_generic" /> +<input type="hidden" name="action" value="{{current_action}}" /> <input type="submit" name="apply" value="Import" /> </form> diff --git a/ishtar_common/templates/admin/town_change_list.html b/ishtar_common/templates/admin/town_change_list.html new file mode 100644 index 000000000..bdbc3e9e3 --- /dev/null +++ b/ishtar_common/templates/admin/town_change_list.html @@ -0,0 +1,24 @@ +{% extends "admin/change_list.html" %} +{% load i18n admin_urls static admin_list %} + + {% block object-tools-items %} + {% if has_add_permission %} + <li> + {% url cl.opts|admin_urlname:'add' as add_url %} + <a href="{% add_preserved_filters add_url is_popup to_field %}" class="addlink"> + {% blocktrans with cl.opts.verbose_name as name %}Add {{ name }}{% endblocktrans %} + </a> + </li> + <li> + <a href="import-from-csv/" class="addlink"> + {% trans "Import from CSV" %} + </a> + </li> + <li> + <a href="import-from-geojson/" class="addlink"> + {% trans "Import from GeoJSON" %} + </a> + </li> + {% endif %} + {% endblock %} + |