Shapefile import

1 files changed, 131 insertions, 7 deletions

diff --git a/chimere/utils.py b/chimere/utils.py
index 268ec3b..3266548 100644
--- a/chimere/utils.py
+++ b/chimere/utils.py
@@ -21,19 +21,25 @@
 Utilitaries
 """
 
+import os
 import tempfile
-import urllib2, re
+import re
+import urllib2
 import unicodedata
 import zipfile
 import StringIO
-from external_utils import OsmApi
+
 from lxml import etree
-from chimere import get_version
+from osgeo import osr
 
 from django.conf import settings
+from django.contrib.gis.gdal import DataSource
 from django.core.exceptions import ObjectDoesNotExist
 from django.utils.translation import ugettext_lazy as _
 
+from chimere import get_version
+from external_utils import OsmApi
+
 def unicode_normalize(string):
     return ''.join(
         (c for c in unicodedata.normalize('NFD', string)
@@ -53,7 +59,7 @@ class ImportManager:
         pass
 
     @classmethod
-    def get_files_inside_zip(cls, zippedfile, suffixes):
+    def get_files_inside_zip(cls, zippedfile, suffixes, dest_dir=None):
         try:
             flz = zipfile.ZipFile(zippedfile)
         except zipfile.BadZipfile:
@@ -71,12 +77,16 @@ class ImportManager:
         files = []
         for filename in filenames:
             if filename:
-                files.append(flz.open(filename))
+                if dest_dir:
+                    files.append(filename)
+                    flz.extract(filename, dest_dir)
+                else:
+                    files.append(flz.open(filename))
             else:
                 files.append(None)
         return files
 
-    def get_source_file(self, source, suffixes):
+    def get_source_file(self, source, suffixes, dest_dir=None):
         if not source:
             try:
                 remotehandle = urllib2.urlopen(self.importer_instance.source)
@@ -92,7 +102,7 @@ class ImportManager:
                 return (None, error.message)
         if self.importer_instance.zipped:
             try:
-                files = self.get_files_inside_zip(source, suffixes)
+                files = self.get_files_inside_zip(source, suffixes, dest_dir)
             except zipfile.BadZipfile:
                 return (None, _(u"Bad zip file"))
             if not files or None in files:
@@ -182,6 +192,120 @@ class KMLManager(ImportManager):
                     m.categories.add(cat)
         return (new_item, updated_item, msg)
 
+class ShapefileManager(ImportManager):
+    u"""
+    Shapefile importer
+    """
+    def __init__(self, importer_instance):
+        self.importer_instance = importer_instance
+
+    def get(self, source=None):
+        u"""
+        Get data from the source
+        Args:
+         - source (None): input file if not provided get it from the distant
+         source provided in the importer instance.
+
+        Return a tuple with:
+         - number of new item ;
+         - number of item updated ;
+         - error detail on error
+        """
+        from models import Marker
+        new_item, updated_item, msg = 0, 0, ''
+        tmpdir = tempfile.mkdtemp()
+        sources, msg = self.get_source_file(source,
+                                            ['.shp', '.dbf', '.prj', '.shx'],
+                                            dest_dir=tmpdir)
+        if msg:
+            return (0, 0, msg)
+        if not sources:
+            return (0, 0, _(u"Error while reading the data source."))
+        # get the srid
+        srid = self.importer_instance.srid
+        if not srid:
+            prjfilename = tmpdir + os.sep + sources[2]
+            with open(prjfilename, 'r') as prj_file:
+                prj_txt = prj_file.read()
+                srs = osr.SpatialReference()
+                srs.ImportFromESRI([prj_txt])
+                srs.AutoIdentifyEPSG()
+                srid = srs.GetAuthorityCode(None)
+            if not srid:
+                # try with the default projection
+                srid = settings.CHIMERE_EPSG_DISPLAY_PROJECTION
+        shapefilename = tmpdir + os.sep + sources[0]
+        ds = DataSource(shapefilename)
+        lyr = ds[0]
+        if lyr.geom_type not in ('Point',):
+            return (0, 0, _(u"Type of geographic item of this shapefile "
+                            u"is not managed by Chimère."))
+        # for this first version it is assumed that the first field is a
+        # id name and the second field is the name
+        id_name = lyr.fields[0] if len(lyr.fields) > 0 else None
+        # test if id_name is well guess
+        if id_name:
+            ids = lyr.get_fields(id_name)
+            if len(ids) != len(set(ids)):
+                id_name = None
+        lbl_name = None
+        if len(lyr.fields) > 1:
+            lbl_name = lyr.fields[1]
+        elif id_name:
+            lbl_name = id_name
+        indexes = []
+        for idx, feat in enumerate(lyr):
+            name = unicode(idx)
+            if lbl_name:
+                name = feat.get(lbl_name)
+                try:
+                    name = unicode(name)
+                except UnicodeDecodeError:
+                    try:
+                        name = unicode(
+                               name.decode(settings.CHIMERE_SHAPEFILE_ENCODING))
+                    except:
+                        continue
+            geom = feat.geom.wkt
+            dct = {'point':'SRID=%s;%s' % (srid, feat.geom.wkt),
+                   'name':name
+                  }
+            m = None
+            if id_name:
+                c_id = feat.get(id_name)
+                dct_import = {
+                    'import_key__icontains':'%s:%s;' % (id_name, c_id),
+                    'import_source':self.importer_instance.source}
+                try:
+                    m = Marker.objects.get(**dct_import)
+                    for k in dct:
+                        setattr(m, k, dct[k])
+                    m.save()
+                    updated_item += 1
+                except ObjectDoesNotExist:
+                    m = None
+                    dct.update({
+                        'import_source':self.importer_instance.source})
+            if not m:
+                dct['status'] = 'I'
+                m = Marker.objects.create(**dct)
+                new_item += 1
+            if id_name:
+                m.set_key(id_name, c_id)
+            m.categories.clear()
+            for cat in self.importer_instance.categories.all():
+                m.categories.add(cat)
+        # clean up
+        tmpdirs = set()
+        for src in sources:
+            dirs = os.sep.join(src.split(os.sep)[:-1])
+            if dirs:
+                tmpdirs.add(tmpdir + os.sep + dirs)
+            os.remove(tmpdir + os.sep + src)
+        for dr in tmpdirs:
+            os.removedirs(dr)
+        return (new_item, updated_item, msg)
+
 
 RE_NODE = re.compile('node\[([^\]]*)\]')