diff options
| -rw-r--r-- | chimere/migrations/0017_auto__add_field_importer_srid.py | 192 | ||||
| -rw-r--r-- | chimere/models.py | 11 | ||||
| -rw-r--r-- | chimere/tests.py | 9 | ||||
| -rw-r--r-- | chimere/utils.py | 138 | ||||
| -rw-r--r-- | example_project/settings.py | 3 |
5 files changed, 343 insertions, 10 deletions
diff --git a/chimere/migrations/0017_auto__add_field_importer_srid.py b/chimere/migrations/0017_auto__add_field_importer_srid.py new file mode 100644 index 0000000..378da72 --- /dev/null +++ b/chimere/migrations/0017_auto__add_field_importer_srid.py @@ -0,0 +1,192 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding field 'Importer.srid' + db.add_column('chimere_importer', 'srid', self.gf('django.db.models.fields.IntegerField')(null=True, blank=True), keep_default=False) + + + def backwards(self, orm): + + # Deleting field 'Importer.srid' + db.delete_column('chimere_importer', 'srid') + + + models = { + 'chimere.area': { + 'Meta': {'ordering': "('order', 'name')", 'object_name': 'Area'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'default': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'lower_right_corner': ('django.contrib.gis.db.models.fields.PointField', [], {'default': "'POINT(0 0)'"}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {}), + 'upper_left_corner': ('django.contrib.gis.db.models.fields.PointField', [], {'default': "'POINT(0 0)'"}), + 'urn': ('django.db.models.fields.SlugField', [], {'db_index': 'True', 'unique': 'True', 'max_length': '50', 'blank': 'True'}) + }, + 'chimere.category': { + 'Meta': {'ordering': "['order']", 'object_name': 'Category'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {}) + }, + 'chimere.color': { + 'Meta': {'ordering': "['order']", 'object_name': 'Color'}, + 'code': ('django.db.models.fields.CharField', [], {'max_length': '6'}), + 'color_theme': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.ColorTheme']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'order': ('django.db.models.fields.IntegerField', [], {}) + }, + 'chimere.colortheme': { + 'Meta': {'object_name': 'ColorTheme'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}) + }, + 'chimere.icon': { + 'Meta': {'object_name': 'Icon'}, + 'height': ('django.db.models.fields.IntegerField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'image': ('django.db.models.fields.files.ImageField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'width': ('django.db.models.fields.IntegerField', [], {}) + }, + 'chimere.importer': { + 'Meta': {'object_name': 'Importer'}, + 'categories': ('chimere.widgets.SelectMultipleField', [], {'to': "orm['chimere.SubCategory']", 'symmetrical': 'False'}), + 'filtr': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'importer_type': ('django.db.models.fields.CharField', [], {'max_length': '4'}), + 'source': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'srid': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'state': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'zipped': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + }, + 'chimere.marker': { + 'Meta': {'ordering': "('status', 'name')", 'object_name': 'Marker'}, + 'categories': ('chimere.widgets.SelectMultipleField', [], {'to': "orm['chimere.SubCategory']", 'symmetrical': 'False'}), + 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'import_key': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'import_source': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'import_version': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'point': ('chimere.widgets.PointField', [], {}), + 'ref_item': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'submited_marker'", 'null': 'True', 'to': "orm['chimere.Marker']"}), + 'route': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'associated_marker'", 'null': 'True', 'to': "orm['chimere.Route']"}), + 'start_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'submiter_comment': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'submiter_email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'submiter_session_key': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}) + }, + 'chimere.multimediafile': { + 'Meta': {'object_name': 'MultimediaFile'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'marker': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'multimedia_files'", 'to': "orm['chimere.Marker']"}), + 'miniature': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'multimedia_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.MultimediaType']"}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {'default': '1'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '200'}) + }, + 'chimere.multimediatype': { + 'Meta': {'object_name': 'MultimediaType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'iframe': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'media_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'mime_type': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}) + }, + 'chimere.news': { + 'Meta': {'object_name': 'News'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'content': ('django.db.models.fields.TextField', [], {}), + 'date': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '150'}) + }, + 'chimere.picturefile': { + 'Meta': {'object_name': 'PictureFile'}, + 'height': ('django.db.models.fields.IntegerField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'marker': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'pictures'", 'to': "orm['chimere.Marker']"}), + 'miniature': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {'default': '1'}), + 'picture': ('django.db.models.fields.files.ImageField', [], {'max_length': '100'}), + 'width': ('django.db.models.fields.IntegerField', [], {}) + }, + 'chimere.property': { + 'Meta': {'object_name': 'Property'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'marker': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Marker']"}), + 'propertymodel': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.PropertyModel']"}), + 'value': ('django.db.models.fields.TextField', [], {}) + }, + 'chimere.propertymodel': { + 'Meta': {'ordering': "('order',)", 'object_name': 'PropertyModel'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {}), + 'type': ('django.db.models.fields.CharField', [], {'max_length': '1'}) + }, + 'chimere.route': { + 'Meta': {'ordering': "('status', 'name')", 'object_name': 'Route'}, + 'associated_file': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.RouteFile']", 'null': 'True', 'blank': 'True'}), + 'categories': ('chimere.widgets.SelectMultipleField', [], {'to': "orm['chimere.SubCategory']", 'symmetrical': 'False'}), + 'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'import_key': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'import_source': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'import_version': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'picture': ('django.db.models.fields.files.ImageField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'ref_item': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'submited_route'", 'null': 'True', 'to': "orm['chimere.Route']"}), + 'route': ('chimere.widgets.RouteField', [], {}), + 'start_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'submiter_comment': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'submiter_email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'submiter_session_key': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}), + 'width': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}) + }, + 'chimere.routefile': { + 'Meta': {'ordering': "('name',)", 'object_name': 'RouteFile'}, + 'file_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'raw_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}), + 'simplified_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}) + }, + 'chimere.subcategory': { + 'Meta': {'ordering': "['category', 'order']", 'object_name': 'SubCategory'}, + 'areas': ('chimere.widgets.SelectMultipleField', [], {'symmetrical': 'False', 'related_name': "'areas'", 'blank': 'True', 'to': "orm['chimere.Area']"}), + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'category': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Category']"}), + 'color_theme': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.ColorTheme']", 'null': 'True', 'blank': 'True'}), + 'icon': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Icon']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'item_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {}) + }, + 'chimere.tinyurl': { + 'Meta': {'object_name': 'TinyUrl'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'parameters': ('django.db.models.fields.CharField', [], {'max_length': '500'}) + } + } + + complete_apps = ['chimere'] diff --git a/chimere/models.py b/chimere/models.py index ed16080..0d8415e 100644 --- a/chimere/models.py +++ b/chimere/models.py @@ -39,7 +39,7 @@ from django.utils.translation import ugettext_lazy as _ from chimere.widgets import PointField, RouteField, SelectMultipleField, \ TextareaWidget from chimere.managers import BaseGeoManager -from chimere.utils import KMLManager, OSMManager +from chimere.utils import KMLManager, OSMManager, ShapefileManager class News(models.Model): """News of the site @@ -193,10 +193,14 @@ class SubCategory(models.Model): in sub_categories.items()] IMPORTERS = {'KML':KMLManager, - 'OSM':OSMManager} + 'OSM':OSMManager, + 'SHP':ShapefileManager + } IMPORTER_CHOICES = (('KML', 'KML'), - ('OSM', 'OSM')) + ('OSM', 'OSM'), + ('SHP', 'Shapefile'), + ) class Importer(models.Model): ''' @@ -213,6 +217,7 @@ class Importer(models.Model): verbose_name=_(u"Associated subcategories")) state = models.CharField(_(u"State"), max_length=200, blank=True, null=True) + srid = models.IntegerField(_(u"SRID"), blank=True, null=True) zipped = models.BooleanField(_(u"Zipped file"), default=False) class Meta: diff --git a/chimere/tests.py b/chimere/tests.py index 6f9a59f..005cfcb 100644 --- a/chimere/tests.py +++ b/chimere/tests.py @@ -87,6 +87,15 @@ class KMLImporterTest(TestCase, ImporterTest): self.marker_importers = [(importer1, 1), (importer2, 2), (importer3, 0), (importer4, 4)] +class ShapefileImporterTest(TestCase, ImporterTest): + def setUp(self): + subcategory_1, subcategory_2 = self._baseSetUp() + importer = Importer.objects.create(importer_type='SHP', + source=test_dir_path+'tests/sample.shp.zip', zipped=True) + importer.categories.add(subcategory_1) + + self.marker_importers = [(importer, 83)] + class OSMImporterTest(TestCase, ImporterTest): def setUp(self): subcategory_1, subcategory_2 = self._baseSetUp() diff --git a/chimere/utils.py b/chimere/utils.py index 268ec3b..3266548 100644 --- a/chimere/utils.py +++ b/chimere/utils.py @@ -21,19 +21,25 @@ Utilitaries """ +import os import tempfile -import urllib2, re +import re +import urllib2 import unicodedata import zipfile import StringIO -from external_utils import OsmApi + from lxml import etree -from chimere import get_version +from osgeo import osr from django.conf import settings +from django.contrib.gis.gdal import DataSource from django.core.exceptions import ObjectDoesNotExist from django.utils.translation import ugettext_lazy as _ +from chimere import get_version +from external_utils import OsmApi + def unicode_normalize(string): return ''.join( (c for c in unicodedata.normalize('NFD', string) @@ -53,7 +59,7 @@ class ImportManager: pass @classmethod - def get_files_inside_zip(cls, zippedfile, suffixes): + def get_files_inside_zip(cls, zippedfile, suffixes, dest_dir=None): try: flz = zipfile.ZipFile(zippedfile) except zipfile.BadZipfile: @@ -71,12 +77,16 @@ class ImportManager: files = [] for filename in filenames: if filename: - files.append(flz.open(filename)) + if dest_dir: + files.append(filename) + flz.extract(filename, dest_dir) + else: + files.append(flz.open(filename)) else: files.append(None) return files - def get_source_file(self, source, suffixes): + def get_source_file(self, source, suffixes, dest_dir=None): if not source: try: remotehandle = urllib2.urlopen(self.importer_instance.source) @@ -92,7 +102,7 @@ class ImportManager: return (None, error.message) if self.importer_instance.zipped: try: - files = self.get_files_inside_zip(source, suffixes) + files = self.get_files_inside_zip(source, suffixes, dest_dir) except zipfile.BadZipfile: return (None, _(u"Bad zip file")) if not files or None in files: @@ -182,6 +192,120 @@ class KMLManager(ImportManager): m.categories.add(cat) return (new_item, updated_item, msg) +class ShapefileManager(ImportManager): + u""" + Shapefile importer + """ + def __init__(self, importer_instance): + self.importer_instance = importer_instance + + def get(self, source=None): + u""" + Get data from the source + Args: + - source (None): input file if not provided get it from the distant + source provided in the importer instance. + + Return a tuple with: + - number of new item ; + - number of item updated ; + - error detail on error + """ + from models import Marker + new_item, updated_item, msg = 0, 0, '' + tmpdir = tempfile.mkdtemp() + sources, msg = self.get_source_file(source, + ['.shp', '.dbf', '.prj', '.shx'], + dest_dir=tmpdir) + if msg: + return (0, 0, msg) + if not sources: + return (0, 0, _(u"Error while reading the data source.")) + # get the srid + srid = self.importer_instance.srid + if not srid: + prjfilename = tmpdir + os.sep + sources[2] + with open(prjfilename, 'r') as prj_file: + prj_txt = prj_file.read() + srs = osr.SpatialReference() + srs.ImportFromESRI([prj_txt]) + srs.AutoIdentifyEPSG() + srid = srs.GetAuthorityCode(None) + if not srid: + # try with the default projection + srid = settings.CHIMERE_EPSG_DISPLAY_PROJECTION + shapefilename = tmpdir + os.sep + sources[0] + ds = DataSource(shapefilename) + lyr = ds[0] + if lyr.geom_type not in ('Point',): + return (0, 0, _(u"Type of geographic item of this shapefile " + u"is not managed by Chimère.")) + # for this first version it is assumed that the first field is a + # id name and the second field is the name + id_name = lyr.fields[0] if len(lyr.fields) > 0 else None + # test if id_name is well guess + if id_name: + ids = lyr.get_fields(id_name) + if len(ids) != len(set(ids)): + id_name = None + lbl_name = None + if len(lyr.fields) > 1: + lbl_name = lyr.fields[1] + elif id_name: + lbl_name = id_name + indexes = [] + for idx, feat in enumerate(lyr): + name = unicode(idx) + if lbl_name: + name = feat.get(lbl_name) + try: + name = unicode(name) + except UnicodeDecodeError: + try: + name = unicode( + name.decode(settings.CHIMERE_SHAPEFILE_ENCODING)) + except: + continue + geom = feat.geom.wkt + dct = {'point':'SRID=%s;%s' % (srid, feat.geom.wkt), + 'name':name + } + m = None + if id_name: + c_id = feat.get(id_name) + dct_import = { + 'import_key__icontains':'%s:%s;' % (id_name, c_id), + 'import_source':self.importer_instance.source} + try: + m = Marker.objects.get(**dct_import) + for k in dct: + setattr(m, k, dct[k]) + m.save() + updated_item += 1 + except ObjectDoesNotExist: + m = None + dct.update({ + 'import_source':self.importer_instance.source}) + if not m: + dct['status'] = 'I' + m = Marker.objects.create(**dct) + new_item += 1 + if id_name: + m.set_key(id_name, c_id) + m.categories.clear() + for cat in self.importer_instance.categories.all(): + m.categories.add(cat) + # clean up + tmpdirs = set() + for src in sources: + dirs = os.sep.join(src.split(os.sep)[:-1]) + if dirs: + tmpdirs.add(tmpdir + os.sep + dirs) + os.remove(tmpdir + os.sep + src) + for dr in tmpdirs: + os.removedirs(dr) + return (new_item, updated_item, msg) + RE_NODE = re.compile('node\[([^\]]*)\]') diff --git a/example_project/settings.py b/example_project/settings.py index 71db794..b0d4188 100644 --- a/example_project/settings.py +++ b/example_project/settings.py @@ -81,6 +81,9 @@ CHIMERE_OSM_API_URL = 'api06.dev.openstreetmap.org' # test URL CHIMERE_OSM_USER = 'test' CHIMERE_OSM_PASSWORD = 'test' +# encoding for shapefile import +CHIMERE_SHAPEFILE_ENCODING = 'ISO-8859-1' + ADMINS = ( # ('Your Name', 'your_email@domain.com'), ) |