Work on KML import

4 files changed, 303 insertions, 15 deletions

diff --git a/chimere/migrations/0008_auto__del_field_importer_subcategory__add_field_route_import_source__a.py b/chimere/migrations/0008_auto__del_field_importer_subcategory__add_field_route_import_source__a.py
new file mode 100644
index 0000000..c40b027
--- /dev/null
+++ b/chimere/migrations/0008_auto__del_field_importer_subcategory__add_field_route_import_source__a.py
@@ -0,0 +1,209 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Deleting field 'Importer.subcategory'
+        db.delete_column('chimere_importer', 'subcategory_id')
+
+        # Adding M2M table for field categories on 'Importer'
+        db.create_table('chimere_importer_categories', (
+            ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
+            ('importer', models.ForeignKey(orm['chimere.importer'], null=False)),
+            ('subcategory', models.ForeignKey(orm['chimere.subcategory'], null=False))
+        ))
+        db.create_unique('chimere_importer_categories', ['importer_id', 'subcategory_id'])
+
+        # Adding field 'Route.import_source'
+        db.add_column('chimere_route', 'import_source', self.gf('django.db.models.fields.CharField')(max_length=200, null=True, blank=True), keep_default=False)
+
+        # Adding field 'Marker.import_source'
+        db.add_column('chimere_marker', 'import_source', self.gf('django.db.models.fields.CharField')(max_length=200, null=True, blank=True), keep_default=False)
+
+
+    def backwards(self, orm):
+        
+        # User chose to not deal with backwards NULL issues for 'Importer.subcategory'
+        raise RuntimeError("Cannot reverse this migration. 'Importer.subcategory' and its values cannot be restored.")
+
+        # Removing M2M table for field categories on 'Importer'
+        db.delete_table('chimere_importer_categories')
+
+        # Deleting field 'Route.import_source'
+        db.delete_column('chimere_route', 'import_source')
+
+        # Deleting field 'Marker.import_source'
+        db.delete_column('chimere_marker', 'import_source')
+
+
+    models = {
+        'chimere.area': {
+            'Meta': {'ordering': "('order', 'name')", 'object_name': 'Area'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'lower_right_corner': ('django.contrib.gis.db.models.fields.PointField', [], {'default': "'POINT(0 0)'"}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'order': ('django.db.models.fields.IntegerField', [], {}),
+            'upper_left_corner': ('django.contrib.gis.db.models.fields.PointField', [], {'default': "'POINT(0 0)'"}),
+            'urn': ('django.db.models.fields.SlugField', [], {'db_index': 'True', 'unique': 'True', 'max_length': '50', 'blank': 'True'})
+        },
+        'chimere.category': {
+            'Meta': {'ordering': "['order']", 'object_name': 'Category'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'order': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'chimere.color': {
+            'Meta': {'ordering': "['order']", 'object_name': 'Color'},
+            'code': ('django.db.models.fields.CharField', [], {'max_length': '6'}),
+            'color_theme': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.ColorTheme']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'order': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'chimere.colortheme': {
+            'Meta': {'object_name': 'ColorTheme'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'})
+        },
+        'chimere.icon': {
+            'Meta': {'object_name': 'Icon'},
+            'height': ('django.db.models.fields.IntegerField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'image': ('django.db.models.fields.files.ImageField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'width': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'chimere.importer': {
+            'Meta': {'object_name': 'Importer'},
+            'categories': ('chimere.widgets.SelectMultipleField', [], {'to': "orm['chimere.SubCategory']", 'symmetrical': 'False'}),
+            'filtr': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'importer_type': ('django.db.models.fields.CharField', [], {'max_length': '4'}),
+            'source_url': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'})
+        },
+        'chimere.marker': {
+            'Meta': {'ordering': "('status', 'name')", 'object_name': 'Marker'},
+            'available_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'categories': ('chimere.widgets.SelectMultipleField', [], {'to': "orm['chimere.SubCategory']", 'symmetrical': 'False'}),
+            'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'import_key': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'import_source': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'multimedia_files': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'marker'", 'null': 'True', 'symmetrical': 'False', 'to': "orm['chimere.MultimediaFile']"}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'pictures': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'related_name': "'marker'", 'null': 'True', 'symmetrical': 'False', 'to': "orm['chimere.PictureFile']"}),
+            'point': ('chimere.widgets.PointField', [], {}),
+            'ref_item': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'submited_marker'", 'null': 'True', 'to': "orm['chimere.Marker']"}),
+            'route': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'associated_marker'", 'null': 'True', 'to': "orm['chimere.Route']"}),
+            'start_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'status': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'submiter_comment': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'submiter_email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}),
+            'submiter_session_key': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'})
+        },
+        'chimere.multimediafile': {
+            'Meta': {'object_name': 'MultimediaFile'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'multimedia_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.MultimediaType']"}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150', 'null': 'True', 'blank': 'True'}),
+            'order': ('django.db.models.fields.IntegerField', [], {'default': '1'}),
+            'url': ('django.db.models.fields.CharField', [], {'max_length': '200'})
+        },
+        'chimere.multimediatype': {
+            'Meta': {'object_name': 'MultimediaType'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'iframe': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'media_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'mime_type': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'})
+        },
+        'chimere.news': {
+            'Meta': {'object_name': 'News'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'content': ('django.db.models.fields.TextField', [], {}),
+            'date': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '150'})
+        },
+        'chimere.picturefile': {
+            'Meta': {'object_name': 'PictureFile'},
+            'height': ('django.db.models.fields.IntegerField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'miniature': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150', 'null': 'True', 'blank': 'True'}),
+            'order': ('django.db.models.fields.IntegerField', [], {'default': '1'}),
+            'picture': ('django.db.models.fields.files.ImageField', [], {'max_length': '100'}),
+            'width': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'chimere.property': {
+            'Meta': {'object_name': 'Property'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'marker': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Marker']"}),
+            'propertymodel': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.PropertyModel']"}),
+            'value': ('django.db.models.fields.TextField', [], {})
+        },
+        'chimere.propertymodel': {
+            'Meta': {'ordering': "('order',)", 'object_name': 'PropertyModel'},
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'order': ('django.db.models.fields.IntegerField', [], {}),
+            'type': ('django.db.models.fields.CharField', [], {'max_length': '1'})
+        },
+        'chimere.route': {
+            'Meta': {'ordering': "('status', 'name')", 'object_name': 'Route'},
+            'associated_file': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.RouteFile']", 'null': 'True', 'blank': 'True'}),
+            'categories': ('chimere.widgets.SelectMultipleField', [], {'to': "orm['chimere.SubCategory']", 'symmetrical': 'False'}),
+            'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'import_key': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'import_source': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'picture': ('django.db.models.fields.files.ImageField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'ref_item': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'submited_route'", 'null': 'True', 'to': "orm['chimere.Route']"}),
+            'route': ('chimere.widgets.RouteField', [], {}),
+            'start_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'status': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'submiter_comment': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'submiter_email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}),
+            'submiter_session_key': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}),
+            'width': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'chimere.routefile': {
+            'Meta': {'ordering': "('name',)", 'object_name': 'RouteFile'},
+            'file_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'raw_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}),
+            'simplified_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'})
+        },
+        'chimere.subcategory': {
+            'Meta': {'ordering': "['category', 'order']", 'object_name': 'SubCategory'},
+            'areas': ('chimere.widgets.SelectMultipleField', [], {'symmetrical': 'False', 'related_name': "'areas'", 'blank': 'True', 'to': "orm['chimere.Area']"}),
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'category': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Category']"}),
+            'color_theme': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.ColorTheme']", 'null': 'True', 'blank': 'True'}),
+            'icon': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Icon']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'item_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}),
+            'order': ('django.db.models.fields.IntegerField', [], {})
+        },
+        'chimere.tinyurl': {
+            'Meta': {'object_name': 'TinyUrl'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'parameters': ('django.db.models.fields.CharField', [], {'max_length': '500'})
+        }
+    }
+
+    complete_apps = ['chimere']
diff --git a/chimere/models.py b/chimere/models.py
index 29c744d..21fdd4e 100644
--- a/chimere/models.py
+++ b/chimere/models.py
@@ -207,12 +207,16 @@ class Importer(models.Model):
                                   blank=True, null=True)
     filtr = models.CharField(_(u"Filter"), max_length=200,
                              blank=True, null=True)
-    subcategory = models.ForeignKey(SubCategory,
-                         verbose_name=_(u"Associated subcategory"))
+    categories = SelectMultipleField(SubCategory,
+                      verbose_name=_(u"Associated subcategories"))
 
     class Meta:
         verbose_name = _(u"Importer")
 
+    @property
+    def manager(self):
+        return IMPORTERS[self.importer_type](self)
+
 class GeographicItem(models.Model):
     name = models.CharField(_(u"Name"), max_length=150)
     categories = SelectMultipleField(SubCategory)
@@ -228,12 +232,12 @@ class GeographicItem(models.Model):
               ('D', _(u'Disabled')),
               ('I', _(u'Imported')),
               ('E', _(u"Excluded")))
-    STATUS_DCT = {}
-    for key, label in STATUS:
-        STATUS_DCT[key] = label
+    STATUS_DCT = dict(STATUS)
     status = models.CharField(_(u"Status"), max_length=1, choices=STATUS)
     import_key = models.CharField(_(u"Import key"), max_length=200,
                                   blank=True, null=True)
+    import_source = models.CharField(_(u"Source"), max_length=200,
+                                     blank=True, null=True)
     if settings.CHIMERE_DAYS_BEFORE_EVENT:
         start_date = models.DateField(_(u"Start date"), blank=True, null=True,
             help_text=_(u"Not mandatory. Set it for dated item such as event. "\
diff --git a/chimere/sample.kml b/chimere/sample.kml
index 1a060d8..665d861 100644
--- a/chimere/sample.kml
+++ b/chimere/sample.kml
@@ -7,7 +7,7 @@
             <name>Category 1</name>
             <Snippet />
             <description>Category description</description>
-            <Placemark>
+            <Placemark id='1'>
                 <name>Place 1</name>
                 <Snippet>snippet</Snippet>
                 <description><![CDATA[<p>Description of place 1 with html</p>]]></description>
@@ -19,14 +19,22 @@
                 <name>Subcategory 1</name>
                 <Snippet />
                 <description><![CDATA[<p>Subcategory description]]></description>
-                <Placemark>
-                    <name>Place 2</name>
+                <Placemark id='2'>
+                    <name>Place 21</name>
                     <Snippet>snippet</Snippet>
-                    <description>Place 2 description</description>
+                    <description>Place 21 description</description>
                     <Point>
                         <coordinates>-4.54328,48.374475,0</coordinates>
                     </Point>
                 </Placemark>
+                <Placemark id='3'>
+                    <name>Place 22</name>
+                    <Snippet>snippet</Snippet>
+                    <description>Place 22 description</description>
+                    <Point>
+                        <coordinates>-4.69242,48.57501,0</coordinates>
+                    </Point>
+                </Placemark>
             </Folder>
         </Folder>
         <Folder>
@@ -37,7 +45,7 @@
                 <name>Subcategory 2</name>
                 <Snippet />
                 <description>Description</description>
-                <Placemark>
+                <Placemark id='4'>
                     <name>Place 3</name>
                     <Snippet>snippet</Snippet>
                     <description><![CDATA[<p>Place 3 description</p>]]></description>
diff --git a/chimere/utils.py b/chimere/utils.py
index 40bc5ad..4e78220 100644
--- a/chimere/utils.py
+++ b/chimere/utils.py
@@ -21,7 +21,10 @@
 Utilitaries
 """
 
-from urllib2 import urlopen, URLError
+import urllib2
+from lxml import etree
+
+from django.core.exceptions import ObjectDoesNotExist
 
 class ImportManager:
     u"""
@@ -39,16 +42,80 @@ class ImportManager:
 class KMLManager(ImportManager):
     u"""
     KML importer/exporter
+    The filtr argument has to be defined as a correct Xpath string pointing
+    to Placemark nodes.
+    A typical XPath string looks like:
+    "//kml:Folder/kml:name[text()='Subcategory 1']/../kml:Placemark"
     """
+    def __init__(self, importer_instance, ns=''):
+        self.importer_instance = importer_instance
+        self.ns = ns
+
     def get(self):
         u"""
         Get data from the source
-        Return a tuple with number of item imported and the detail
+        Return a tuple with:
+        - number of new item ;
+        - number of item updated ;
+        - detail
         """
+        from models import Marker
+        new_item, updated_item, msg = 0, 0, ''
         try:
-            source = urlopen(self.importer_instance.source_url)
-        except URLError as error:
-            return (0, error.reason)
+            source = urllib2.urlopen(self.importer_instance.source_url)
+        except ValueError:
+            # assume it is a local file
+            try:
+                source = open(self.importer_instance.source_url)
+            except IOError, msg:
+                return (new_item, updated_item, msg)
+        except urllib2.URLError as error:
+            return (new_item, updated_item, error.reason)
+        tree = etree.parse(source)
+        # try to get default namespace
+        if not self.ns:
+            self.ns = tree.getroot().nsmap[None]
+        for placemark in tree.xpath(self.importer_instance.filtr,
+                                    namespaces={'kml':self.ns}):
+            name, point, linestring = None, None, None
+            pl_id = placemark.attrib.get('id')
+            ns = '{%s}' % self.ns
+            for item in placemark:
+                if item.tag == ns + 'name':
+                    name = item.text
+                elif item.tag == ns + 'description':
+                    description = item.text
+                elif item.tag == ns + 'Point':
+                    for coord in item:
+                        if coord.tag == ns + 'coordinates':
+                            x, y, z = coord.text.split(',')
+                            point = 'SRID=4326;POINT(%s %s)' % (x, y)
+            if point:
+                dct = {'point':point,
+                       'description':description,
+                       'name':name,}
+                m = None
+                if pl_id:
+                    dct_import = {
+                        'import_key':pl_id,
+                        'import_source':self.importer_instance.source_url}
+                    try:
+                        m = Marker.objects.get(**dct_import)
+                        for k in dct:
+                            setattr(m, k, dct[k])
+                        m.save()
+                        updated_item += 1
+                    except ObjectDoesNotExist:
+                        m = None
+                        dct.update(dct_import)
+                if not m:
+                    dct['status'] = 'I'
+                    m = Marker.objects.create(**dct)
+                    new_item += 1
+                m.categories.clear()
+                for cat in self.importer_instance.categories.all():
+                    m.categories.add(cat)
+        return (new_item, updated_item, msg)
 
 class OSMManager(ImportManager):
     pass