Manage zipfile for imports

8 files changed, 270 insertions, 29 deletions

diff --git a/chimere/migrations/0016_auto__add_field_importer_zipped.py b/chimere/migrations/0016_auto__add_field_importer_zipped.py
new file mode 100644
index 0000000..9df4d91
--- /dev/null
+++ b/chimere/migrations/0016_auto__add_field_importer_zipped.py
@@ -0,0 +1,191 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding field 'Importer.zipped'
+        db.add_column('chimere_importer', 'zipped', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=False)
+
+
+    def backwards(self, orm):
+        
+        # Deleting field 'Importer.zipped'
+        db.delete_column('chimere_importer', 'zipped')
+
+
+    models = {
+        'chimere.area': {
+            'Meta': {'ordering': "('order', 'name')", 'object_name': 'Area'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'default': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'lower_right_corner': ('django.contrib.gis.db.models.fields.PointField', [], {'default': "'POINT(0 0)'"}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'order': ('django.db.models.fields.IntegerField', [], {}),
+            'upper_left_corner': ('django.contrib.gis.db.models.fields.PointField', [], {'default': "'POINT(0 0)'"}),
+            'urn': ('django.db.models.fields.SlugField', [], {'db_index': 'True', 'unique': 'True', 'max_length': '50', 'blank': 'True'})
+        },
+        'chimere.category': {
+            'Meta': {'ordering': "['order']", 'object_name': 'Category'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'order': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'chimere.color': {
+            'Meta': {'ordering': "['order']", 'object_name': 'Color'},
+            'code': ('django.db.models.fields.CharField', [], {'max_length': '6'}),
+            'color_theme': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.ColorTheme']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'order': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'chimere.colortheme': {
+            'Meta': {'object_name': 'ColorTheme'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'})
+        },
+        'chimere.icon': {
+            'Meta': {'object_name': 'Icon'},
+            'height': ('django.db.models.fields.IntegerField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'image': ('django.db.models.fields.files.ImageField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'width': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'chimere.importer': {
+            'Meta': {'object_name': 'Importer'},
+            'categories': ('chimere.widgets.SelectMultipleField', [], {'to': "orm['chimere.SubCategory']", 'symmetrical': 'False'}),
+            'filtr': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'importer_type': ('django.db.models.fields.CharField', [], {'max_length': '4'}),
+            'source': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'state': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'zipped': ('django.db.models.fields.BooleanField', [], {'default': 'False'})
+        },
+        'chimere.marker': {
+            'Meta': {'ordering': "('status', 'name')", 'object_name': 'Marker'},
+            'categories': ('chimere.widgets.SelectMultipleField', [], {'to': "orm['chimere.SubCategory']", 'symmetrical': 'False'}),
+            'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'import_key': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'import_source': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'import_version': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'point': ('chimere.widgets.PointField', [], {}),
+            'ref_item': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'submited_marker'", 'null': 'True', 'to': "orm['chimere.Marker']"}),
+            'route': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'associated_marker'", 'null': 'True', 'to': "orm['chimere.Route']"}),
+            'start_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'status': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'submiter_comment': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'submiter_email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}),
+            'submiter_session_key': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'})
+        },
+        'chimere.multimediafile': {
+            'Meta': {'object_name': 'MultimediaFile'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'marker': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'multimedia_files'", 'to': "orm['chimere.Marker']"}),
+            'miniature': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'multimedia_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.MultimediaType']"}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'order': ('django.db.models.fields.IntegerField', [], {'default': '1'}),
+            'url': ('django.db.models.fields.URLField', [], {'max_length': '200'})
+        },
+        'chimere.multimediatype': {
+            'Meta': {'object_name': 'MultimediaType'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'iframe': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'media_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'mime_type': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'})
+        },
+        'chimere.news': {
+            'Meta': {'object_name': 'News'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'content': ('django.db.models.fields.TextField', [], {}),
+            'date': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '150'})
+        },
+        'chimere.picturefile': {
+            'Meta': {'object_name': 'PictureFile'},
+            'height': ('django.db.models.fields.IntegerField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'marker': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'pictures'", 'to': "orm['chimere.Marker']"}),
+            'miniature': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'order': ('django.db.models.fields.IntegerField', [], {'default': '1'}),
+            'picture': ('django.db.models.fields.files.ImageField', [], {'max_length': '100'}),
+            'width': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'chimere.property': {
+            'Meta': {'object_name': 'Property'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'marker': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Marker']"}),
+            'propertymodel': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.PropertyModel']"}),
+            'value': ('django.db.models.fields.TextField', [], {})
+        },
+        'chimere.propertymodel': {
+            'Meta': {'ordering': "('order',)", 'object_name': 'PropertyModel'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'order': ('django.db.models.fields.IntegerField', [], {}),
+            'type': ('django.db.models.fields.CharField', [], {'max_length': '1'})
+        },
+        'chimere.route': {
+            'Meta': {'ordering': "('status', 'name')", 'object_name': 'Route'},
+            'associated_file': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.RouteFile']", 'null': 'True', 'blank': 'True'}),
+            'categories': ('chimere.widgets.SelectMultipleField', [], {'to': "orm['chimere.SubCategory']", 'symmetrical': 'False'}),
+            'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'import_key': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'import_source': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'import_version': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'picture': ('django.db.models.fields.files.ImageField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'ref_item': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'submited_route'", 'null': 'True', 'to': "orm['chimere.Route']"}),
+            'route': ('chimere.widgets.RouteField', [], {}),
+            'start_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'status': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'submiter_comment': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'submiter_email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}),
+            'submiter_session_key': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}),
+            'width': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'chimere.routefile': {
+            'Meta': {'ordering': "('name',)", 'object_name': 'RouteFile'},
+            'file_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'raw_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}),
+            'simplified_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'})
+        },
+        'chimere.subcategory': {
+            'Meta': {'ordering': "['category', 'order']", 'object_name': 'SubCategory'},
+            'areas': ('chimere.widgets.SelectMultipleField', [], {'symmetrical': 'False', 'related_name': "'areas'", 'blank': 'True', 'to': "orm['chimere.Area']"}),
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'category': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Category']"}),
+            'color_theme': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.ColorTheme']", 'null': 'True', 'blank': 'True'}),
+            'icon': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Icon']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'item_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'order': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'chimere.tinyurl': {
+            'Meta': {'object_name': 'TinyUrl'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'parameters': ('django.db.models.fields.CharField', [], {'max_length': '500'})
+        }
+    }
+
+    complete_apps = ['chimere']
diff --git a/chimere/models.py b/chimere/models.py
index 18ea394..ed16080 100644
--- a/chimere/models.py
+++ b/chimere/models.py
@@ -213,6 +213,7 @@ class Importer(models.Model):
                       verbose_name=_(u"Associated subcategories"))
     state = models.CharField(_(u"State"), max_length=200,
                              blank=True, null=True)
+    zipped = models.BooleanField(_(u"Zipped file"), default=False)
 
     class Meta:
         verbose_name = _(u"Importer")
diff --git a/chimere/templates/chimere/blocks/welcome.html b/chimere/templates/chimere/blocks/welcome.html
index 939e5f0..6751f14 100644
--- a/chimere/templates/chimere/blocks/welcome.html
+++ b/chimere/templates/chimere/blocks/welcome.html
@@ -19,7 +19,7 @@
                        {% else %}
                             <h3>{{news.name}} &ndash; {{ news.start_date }}{% if news.end_date %} - {{ news.end_date }}{% endif %}</h3>
                             {% for property in news.getProperties %}
-                                <p id='{{news.propertymodel.getNamedId}}'>{{ property.value|sanitize:"p b i br hr strong em span:style a:href:target ul li ol h1 h2 h3 h4"|safe }}</p>
+                                <p id='{{news.propertymodel.getNamedId}}'>{{ property.value|sanitize:"p b i br hr strong em span:style a:href:target ul li ol h1 h2 h3 h4 table td th tr"|safe }}</p>
                             {% endfor %}
                             <p class='marker_link'><a href='{{ news.get_absolute_url }}'>{% trans "See it on the map"%}</a></p>
                         {% endif %}
diff --git a/chimere/templates/chimere/detail.html b/chimere/templates/chimere/detail.html
index 6c9f27c..faf6db4 100644
--- a/chimere/templates/chimere/detail.html
+++ b/chimere/templates/chimere/detail.html
@@ -15,10 +15,10 @@
     {% if marker.end_date %} - {{marker.end_date|date:"D d M Y"}}</p>{% endif %}</span>
     {% endif %}
     {% if marker.description %}
-    <p class='description'>{{ marker.description|sanitize:"p b i br hr strong em span:style a:href:target ul li ol h1 h2 h3 h4"|safe}}</p>
+    <p class='description'>{{ marker.description|sanitize:"p b i br hr strong em span:style a:href:target ul li ol h1 h2 h3 h4 table td tr th"|safe}}</p>
     {% endif %}
     {% for property in marker.getProperties %}
-    <p id='{{property.propertymodel.getNamedId}}'>{{ property.value|sanitize:"p b i br hr strong em span:style a:href:target ul li ol h1 h2 h3 h4"|safe}}</p>
+    <p id='{{property.propertymodel.getNamedId}}'>{{ property.value|sanitize:"p b i br hr strong em span:style a:href:target ul li ol h1 h2 h3 h4 table td tr th"|safe}}</p>
     {% endfor %}
     {% if marker.multimedia_items %}
     <a href='#' id='show_gallery_link'>{% trans "Show multimedia gallery" %}</a>
diff --git a/chimere/tests.py b/chimere/tests.py
index 7d84394..6f9a59f 100644
--- a/chimere/tests.py
+++ b/chimere/tests.py
@@ -65,22 +65,27 @@ class KMLImporterTest(TestCase, ImporterTest):
     def setUp(self):
         subcategory_1, subcategory_2 = self._baseSetUp()
         importer1 = Importer.objects.create(importer_type='KML',
-            source=test_dir_path+'sample.kml',
+            source=test_dir_path+'tests/sample.kml',
             filtr="Category 1")
         importer1.categories.add(subcategory_1)
 
         importer2 = Importer.objects.create(importer_type='KML',
-         source=test_dir_path+'sample.kml',
+         source=test_dir_path+'tests/sample.kml',
          filtr="Subcategory 1")
         importer2.categories.add(subcategory_1)
         importer2.categories.add(subcategory_2)
 
         importer3 = Importer.objects.create(importer_type='KML',
-         source=test_path+'sample.kml',
+         source=test_path+'tests/sample.kml',
          filtr="Subcategory 3")
         importer3.categories.add(subcategory_2)
 
-        self.marker_importers = [(importer1, 1), (importer2, 2), (importer3, 0)]
+        importer4 = Importer.objects.create(importer_type='KML',
+         source=test_dir_path+'tests/sample.kml.zip', zipped=True)
+        importer4.categories.add(subcategory_1)
+
+        self.marker_importers = [(importer1, 1), (importer2, 2), (importer3, 0),
+                                 (importer4, 4)]
 
 class OSMImporterTest(TestCase, ImporterTest):
     def setUp(self):
diff --git a/chimere/sample.kml b/chimere/tests/sample.kml
index 665d861..665d861 100644
--- a/chimere/sample.kml
+++ b/chimere/tests/sample.kml
diff --git a/chimere/tests/sample.kml.zip b/chimere/tests/sample.kml.zip
new file mode 100644
index 0000000..f5ee093
--- /dev/null
+++ b/chimere/tests/sample.kml.zip
diff --git a/chimere/utils.py b/chimere/utils.py
index 97f710f..2459075 100644
--- a/chimere/utils.py
+++ b/chimere/utils.py
@@ -23,6 +23,8 @@ Utilitaries
 
 import urllib2, re
 import unicodedata
+import zipfile
+import StringIO
 from external_utils import OsmApi
 from lxml import etree
 from chimere import get_version
@@ -36,6 +38,27 @@ def unicode_normalize(string):
         (c for c in unicodedata.normalize('NFD', string)
         if unicodedata.category(c) != 'Mn'))
 
+def get_files_inside_zip(zippedfile, suffixes):
+    try:
+        flz = zipfile.ZipFile(zippedfile)
+    except zipfile.BadZipfile:
+        return [], _(u"Bad zip file")
+    namelist = flz.namelist()
+    filenames = []
+    for suffix in suffixes:
+        current_file_name = None
+        for name in namelist:
+            if name.endswith(suffix):
+                current_file_name = name
+        filenames.append(current_file_name)
+    files = []
+    for filename in filenames:
+        if filename:
+            files.append(flz.open(filename))
+        else:
+            files.append(None)
+    return files
+
 class ImportManager:
     u"""
     Generic class for specific importers
@@ -56,35 +79,54 @@ class KMLManager(ImportManager):
     imported
     """
     XPATH = '//kml:Folder/kml:name[text()="%s"]/../kml:Placemark'
+    DEFAULT_XPATH = '//kml:Placemark'
     def __init__(self, importer_instance, ns=''):
         self.importer_instance = importer_instance
         self.ns = ns
 
-    def get(self):
+    def get(self, source=None):
         u"""
         Get data from the source
+        Args:
+         - source (None): input file if not provided get it from the distant
+         source provided in the importer instance.
+
         Return a tuple with:
-        - number of new item ;
-        - number of item updated ;
-        - error detail on error
+         - number of new item ;
+         - number of item updated ;
+         - error detail on error
         """
         from models import Marker
         new_item, updated_item, msg = 0, 0, ''
-        try:
-            source = urllib2.urlopen(self.importer_instance.source)
-        except ValueError:
-            # assume it is a local file
+        if not source:
             try:
-                source = open(self.importer_instance.source)
-            except IOError, msg:
-                return (new_item, updated_item, msg)
-        except urllib2.URLError as error:
-            return (new_item, updated_item, error.message)
+                remotehandle = urllib2.urlopen(self.importer_instance.source)
+                source = StringIO.StringIO(remotehandle.read())
+                remotehandle.close()
+            except ValueError:
+                # assume it is a local file
+                try:
+                    source = open(self.importer_instance.source)
+                except IOError, msg:
+                    return (new_item, updated_item, msg)
+            except urllib2.URLError as error:
+                return (new_item, updated_item, error.message)
+        if self.importer_instance.zipped:
+            try:
+                files = get_files_inside_zip(source, ['.kml'])
+            except zipfile.BadZipfile:
+                return (new_item, updated_item, _(u"Bad zip file"))
+            if not files or not files[0]:
+                return (new_item, updated_item,
+                        _(u"No KML file inside the zip file"))
+            source = files[0]
         tree = etree.parse(source)
         # try to get default namespace
         if not self.ns:
             self.ns = tree.getroot().nsmap[None]
-        for placemark in tree.xpath(self.XPATH % self.importer_instance.filtr,
+        xpath = self.XPATH % self.importer_instance.filtr \
+                  if self.importer_instance.filtr else self.DEFAULT_XPATH
+        for placemark in tree.xpath(xpath,
                                     namespaces={'kml':self.ns}):
             name, point, linestring = None, None, None
             pl_id = placemark.attrib.get('id')
@@ -141,13 +183,17 @@ class OSMManager(ImportManager):
     The filtr argument is XAPI args or empty if it is an OSM file.
     """
 
-    def get(self, get_items=None):
+    def get(self, source=None):
         u"""
         Get data from the source
+        Args:
+         - source (None): input file if not provided get it from the distant
+         source provided in the importer instance.
+
         Return a tuple with:
-        - new items or number of new items if nb_only = True;
-        - updated items or number of updated items if nb_only = True;
-        - error detail on error
+        - new items;
+        - updated items;
+        - error detail on error.
         """
         from models import Marker
         new_item, updated_item = 0 , 0
@@ -208,15 +254,13 @@ class OSMManager(ImportManager):
                 m.categories.clear()
                 for cat in self.importer_instance.categories.all():
                     m.categories.add(cat)
-        if not get_items:
-            return (new_item, updated_item, msg)
-        return (items, new_item, updated_item, msg)
+        return (new_item, updated_item, msg)
 
     def put(self):
         # first of all: reimport in order to verify that no changes has been
         # made since the last import
         from models import Marker
-        items, new_item, updated_item, msg = self.get(get_items=True)
+        new_item, updated_item, msg = self.get()
         # check if import is possible
         if msg:
             return 0, msg