diff options
Diffstat (limited to 'chimere/utils.py')
| -rw-r--r-- | chimere/utils.py | 138 |
1 files changed, 131 insertions, 7 deletions
diff --git a/chimere/utils.py b/chimere/utils.py index 268ec3b..3266548 100644 --- a/chimere/utils.py +++ b/chimere/utils.py @@ -21,19 +21,25 @@ Utilitaries """ +import os import tempfile -import urllib2, re +import re +import urllib2 import unicodedata import zipfile import StringIO -from external_utils import OsmApi + from lxml import etree -from chimere import get_version +from osgeo import osr from django.conf import settings +from django.contrib.gis.gdal import DataSource from django.core.exceptions import ObjectDoesNotExist from django.utils.translation import ugettext_lazy as _ +from chimere import get_version +from external_utils import OsmApi + def unicode_normalize(string): return ''.join( (c for c in unicodedata.normalize('NFD', string) @@ -53,7 +59,7 @@ class ImportManager: pass @classmethod - def get_files_inside_zip(cls, zippedfile, suffixes): + def get_files_inside_zip(cls, zippedfile, suffixes, dest_dir=None): try: flz = zipfile.ZipFile(zippedfile) except zipfile.BadZipfile: @@ -71,12 +77,16 @@ class ImportManager: files = [] for filename in filenames: if filename: - files.append(flz.open(filename)) + if dest_dir: + files.append(filename) + flz.extract(filename, dest_dir) + else: + files.append(flz.open(filename)) else: files.append(None) return files - def get_source_file(self, source, suffixes): + def get_source_file(self, source, suffixes, dest_dir=None): if not source: try: remotehandle = urllib2.urlopen(self.importer_instance.source) @@ -92,7 +102,7 @@ class ImportManager: return (None, error.message) if self.importer_instance.zipped: try: - files = self.get_files_inside_zip(source, suffixes) + files = self.get_files_inside_zip(source, suffixes, dest_dir) except zipfile.BadZipfile: return (None, _(u"Bad zip file")) if not files or None in files: @@ -182,6 +192,120 @@ class KMLManager(ImportManager): m.categories.add(cat) return (new_item, updated_item, msg) +class ShapefileManager(ImportManager): + u""" + Shapefile importer + """ + def __init__(self, importer_instance): + self.importer_instance = importer_instance + + def get(self, source=None): + u""" + Get data from the source + Args: + - source (None): input file if not provided get it from the distant + source provided in the importer instance. + + Return a tuple with: + - number of new item ; + - number of item updated ; + - error detail on error + """ + from models import Marker + new_item, updated_item, msg = 0, 0, '' + tmpdir = tempfile.mkdtemp() + sources, msg = self.get_source_file(source, + ['.shp', '.dbf', '.prj', '.shx'], + dest_dir=tmpdir) + if msg: + return (0, 0, msg) + if not sources: + return (0, 0, _(u"Error while reading the data source.")) + # get the srid + srid = self.importer_instance.srid + if not srid: + prjfilename = tmpdir + os.sep + sources[2] + with open(prjfilename, 'r') as prj_file: + prj_txt = prj_file.read() + srs = osr.SpatialReference() + srs.ImportFromESRI([prj_txt]) + srs.AutoIdentifyEPSG() + srid = srs.GetAuthorityCode(None) + if not srid: + # try with the default projection + srid = settings.CHIMERE_EPSG_DISPLAY_PROJECTION + shapefilename = tmpdir + os.sep + sources[0] + ds = DataSource(shapefilename) + lyr = ds[0] + if lyr.geom_type not in ('Point',): + return (0, 0, _(u"Type of geographic item of this shapefile " + u"is not managed by Chimère.")) + # for this first version it is assumed that the first field is a + # id name and the second field is the name + id_name = lyr.fields[0] if len(lyr.fields) > 0 else None + # test if id_name is well guess + if id_name: + ids = lyr.get_fields(id_name) + if len(ids) != len(set(ids)): + id_name = None + lbl_name = None + if len(lyr.fields) > 1: + lbl_name = lyr.fields[1] + elif id_name: + lbl_name = id_name + indexes = [] + for idx, feat in enumerate(lyr): + name = unicode(idx) + if lbl_name: + name = feat.get(lbl_name) + try: + name = unicode(name) + except UnicodeDecodeError: + try: + name = unicode( + name.decode(settings.CHIMERE_SHAPEFILE_ENCODING)) + except: + continue + geom = feat.geom.wkt + dct = {'point':'SRID=%s;%s' % (srid, feat.geom.wkt), + 'name':name + } + m = None + if id_name: + c_id = feat.get(id_name) + dct_import = { + 'import_key__icontains':'%s:%s;' % (id_name, c_id), + 'import_source':self.importer_instance.source} + try: + m = Marker.objects.get(**dct_import) + for k in dct: + setattr(m, k, dct[k]) + m.save() + updated_item += 1 + except ObjectDoesNotExist: + m = None + dct.update({ + 'import_source':self.importer_instance.source}) + if not m: + dct['status'] = 'I' + m = Marker.objects.create(**dct) + new_item += 1 + if id_name: + m.set_key(id_name, c_id) + m.categories.clear() + for cat in self.importer_instance.categories.all(): + m.categories.add(cat) + # clean up + tmpdirs = set() + for src in sources: + dirs = os.sep.join(src.split(os.sep)[:-1]) + if dirs: + tmpdirs.add(tmpdir + os.sep + dirs) + os.remove(tmpdir + os.sep + src) + for dr in tmpdirs: + os.removedirs(dr) + return (new_item, updated_item, msg) + RE_NODE = re.compile('node\[([^\]]*)\]') |