Merge branch 'master' into saclay

Conflicts:
	chimere/admin.py
	chimere/forms.py
	example_project/settings.py

1 files changed, 203 insertions, 18 deletions

diff --git a/chimere/utils.py b/chimere/utils.py
index d61ad19..a24e22f 100644
--- a/chimere/utils.py
+++ b/chimere/utils.py
@@ -21,6 +21,7 @@
 Utilitaries
 """
 
+import csv
 import datetime
 import os
 import re
@@ -429,9 +430,161 @@ class ShapefileManager(ImportManager):
         buff.close()
         return filename, zip_stream
 
-RE_NODE = re.compile('node\[([^\]]*)\]')
+class ImportManager:
+    u"""
+    Generic class for specific importers
+    """
+    default_source = None
+    def __init__(self, importer_instance):
+        self.importer_instance = importer_instance
+        self.default_name = " - ".join([cat.name
+           for cat in self.importer_instance.categories.order_by('name').all()])
+
+    def get(self):
+        pass
+
+    def put(self):
+        pass
+
+    def create_or_update_item(self, cls, values, import_key, version=None):
+        updated, created, item = False, False, None
+        if import_key:
+            dct_import = {
+                'import_key__icontains':'%s:%s;' % (
+                        self.importer_instance.importer_type,
+                        import_key),
+                'import_source':self.importer_instance.source}
+            try:
+                item = cls.objects.get(**dct_import)
+                if version and item.import_version == int(version):
+                    # no update since the last import
+                    return item, None, None
+                for k in values:
+                    setattr(item, k, values[k])
+                try:
+                    item.save()
+                except TypeError:
+                    # error on data source
+                    return None, False, False
+                updated = True
+            except ObjectDoesNotExist:
+                pass
+        if not item:
+            values.update({
+                'import_source':self.importer_instance.source})
+            values['status'] = 'I'
+            try:
+                item = cls.objects.create(**values)
+            except TypeError:
+                # error on data source
+                return None, False, False
+            created = True
+        if import_key:
+            item.set_key(self.importer_instance.importer_type,
+                         import_key)
+        item.categories.clear()
+        for cat in self.importer_instance.categories.all():
+            item.categories.add(cat)
+        return item, updated, created
+
+    @classmethod
+    def get_files_inside_zip(cls, zippedfile, suffixes, dest_dir=None):
+        try:
+            flz = zipfile.ZipFile(zippedfile)
+        except zipfile.BadZipfile:
+            return [], _(u"Bad zip file")
+        namelist = flz.namelist()
+        filenames = []
+        for suffix in suffixes:
+            current_file_name = None
+            for name in namelist:
+                if name.endswith(suffix) \
+                  or name.endswith(suffix.lower()) \
+                  or name.endswith(suffix.upper()):
+                    current_file_name = name
+            filenames.append(current_file_name)
+        files = []
+        for filename in filenames:
+            if filename:
+                if dest_dir:
+                    files.append(filename)
+                    flz.extract(filename, dest_dir)
+                else:
+                    files.append(flz.open(filename))
+            else:
+                files.append(None)
+        return files
 
-# manage deleted item from OSM
+    def get_source_file(self, source, suffixes, dest_dir=None,
+                        extra_url=None):
+        if not hasattr(source, 'read'):
+            if not source:
+                source = self.importer_instance.source \
+                       if self.importer_instance.source else self.default_source
+            try:
+                url = source
+                if extra_url:
+                    url += extra_url
+                remotehandle = urllib2.urlopen(url)
+                source = StringIO.StringIO(remotehandle.read())
+                remotehandle.close()
+            except ValueError:
+                # assume it is a local file
+                try:
+                    source = open(source)
+                except IOError, msg:
+                    return (None, msg)
+            except urllib2.URLError as error:
+                return (None, error.message)
+        if self.importer_instance.zipped:
+            try:
+                files = self.get_files_inside_zip(source, suffixes, dest_dir)
+            except zipfile.BadZipfile:
+                return (None, _(u"Bad zip file"))
+            if not files or None in files:
+                return (None,
+                        _(u"Missing file(s) inside the zip file"))
+            source = files[0] if len(suffixes) == 1 else files
+        return (source, None)
+
+class CSVManager(ImportManager):
+    u"""
+    CSV importer
+    """
+    _COLS = [("Id", 'pk'), (_(u"Name"), 'name'),
+            (_(u"Categories"), lambda obj:", ".join(
+                                [c.name for c in obj.categories.all()])),
+            (_(u"State"), 'status')]
+    COLS = {'marker':_COLS+[(_(u"Description"), 'description'),
+                            (_(u"Localisation"), lambda obj: obj.point.wkt)],
+            'route':_COLS+[(_(u"Localisation"), lambda obj: obj.route.wkt)]}
+
+    @classmethod
+    def export(cls, queryset):
+        dct = {'description':unicode(datetime.date.today()), 'data':[]}
+        cls_name = queryset.model.__name__.lower()
+        cols = cls.COLS[cls_name][:]
+        for pm in queryset.model.properties():
+            cols.append((pm.name, pm.getAttrName()))
+        header = [lbl for lbl, attr in cols]
+        dct['data'].append(header)
+        for item in queryset.all():
+            data = []
+            for lbl, attr in cols:
+                if callable(attr):
+                    data.append(attr(item))
+                    print lbl, attr(item)
+                else:
+                    data.append(getattr(item, attr))
+            dct['data'].append(data)
+        filename = unicode_normalize(settings.PROJECT_NAME + dct['description']\
+                                     + '.csv')
+        result = render_to_response('chimere/export_%s.csv' % cls_name, dct)
+        return filename, result
+
+RE_HOOK = re.compile('\[([^\]]*)\]')
+
+# TODO: manage deleted item from OSM
 
 class OSMManager(ImportManager):
     u"""
@@ -554,26 +707,58 @@ class OSMManager(ImportManager):
                      password=settings.CHIMERE_OSM_PASSWORD)
         api.ChangesetCreate({u"comment": u"Import from Chimère %s" % \
                                                             get_version()})
-        tag = RE_NODE.finddall(self.importer_instance.filtr)
-        if not tag:
+        hooks = RE_HOOK.findall(self.importer_instance.filtr)
+        if not hooks:
             return 0, _(u"Bad param")
-        tag = tag[0].split('=')
-        default_dct = {'tag':{tag[0]:tag[1]},
+        tags = {}
+        bbox = []
+        for hook in hooks:
+            key, value = hook.split('=')
+            if '*' in value or '|' in key or '|' in value:
+                continue
+            if key == 'bbox':
+                x1, y1, x2, y2 = [float(val) for val in value.split(',')]
+                bbox =  GEOSGeometry(
+                    'POLYGON((%f %f,%f %f,%f %f,%f %f,%f %f))' % (
+                    x1, y1, x2, y1, x2, y2, x1, y2, x1, y1), srid=4326)
+                continue
+            tags[key] = value
+        if not tags:
+            return 0, _(u"No non ambigious tag is defined in the XAPI request")
+        if not bbox:
+            return 0, _(u"No bounding box is defined in the XAPI request."\
+            u"If you are sure to manage the entire planet set the bounding box"\
+            u" to -180,-90,180,90")
+        default_dct = {'tag':tags,
                        'import_source':self.importer_instance.source}
-        for idx, item in Marker.objects.filter(status='A',
-                categories=self.importer_instance.categories.all()):
-            dct = default_dct.update({
-                        'name':item.name,
-                        'lon':item.point.lon,
-                        'lat':item.point.lat})
+        idx = -1
+        for idx, item in enumerate(Marker.objects.filter(status='A',
+                point__contained=bbox,
+                categories=self.importer_instance.categories.all(),
+                not_for_osm=False, modified_since_import=True).all()):
+            dct = default_dct.copy()
+            dct.update({'lon':item.point.x,
+                        'lat':item.point.y})
+            dct['tag']['name'] = item.name
             node = None
-            import_key = marker.get_key('OSM')
-            if not import_key:
-                node = OsmApi.NodeCreate(dct)
+            import_key = item.get_key('OSM')
+            updated = False
+            if import_key:
+                try:
+                    dct['id'] = import_key
+                    dct['version'] = item.import_version
+                    node = api.NodeUpdate(dct)
+                    updated = True
+                except OsmApi.ApiError, error:
+                    if error.status == 404:
+                        dct.pop('id')
+                        dct.pop('version')
+                        pass # if the node doesn't exist it is created
+                    else:
+                        raise
+            if not updated:
+                node = api.NodeCreate(dct)
                 item.set_key('OSM', node['id'])
-            else:
-                dct['id'] = import_key
-                node = OsmApi.NodeUpdate(dct)
             item.import_version = node['version']
             item.save()
         api.ChangesetClose()