diff options
| -rw-r--r-- | chimere/admin.py | 17 | ||||
| -rw-r--r-- | chimere/forms.py | 2 | ||||
| -rw-r--r-- | chimere/migrations/0029_auto__add_field_marker_modified_since_import__add_field_marker_not_for.py | 249 | ||||
| -rw-r--r-- | chimere/models.py | 55 | ||||
| -rw-r--r-- | chimere/tasks.py | 3 | ||||
| -rw-r--r-- | chimere/templates/chimere/export_marker.csv | 2 | ||||
| -rw-r--r-- | chimere/utils.py | 221 | ||||
| -rw-r--r-- | example_project/settings.py | 1 |
8 files changed, 524 insertions, 26 deletions
diff --git a/chimere/admin.py b/chimere/admin.py index e86bcea..d497da5 100644 --- a/chimere/admin.py +++ b/chimere/admin.py @@ -41,7 +41,8 @@ from chimere.models import Category, Icon, SubCategory, Marker, \ PropertyModel, News, Route, Area, ColorTheme, Color, RouteFile,\ MultimediaType, MultimediaFile, PictureFile, Importer, Layer, AreaLayers,\ PropertyModelChoice, MultimediaExtension, Page -from chimere.utils import unicode_normalize, ShapefileManager, KMLManager +from chimere.utils import unicode_normalize, ShapefileManager, KMLManager,\ + CSVManager def get_areas_for_user(user): """ @@ -87,6 +88,16 @@ def export_to_shapefile(modeladmin, request, queryset): return response export_to_shapefile.short_description = _(u"Export to Shapefile") +def export_to_csv(modeladmin, request, queryset): + u""" + Export data to CSV + """ + filename, result = CSVManager.export(queryset) + response = HttpResponse(result, mimetype='text/csv') + response['Content-Disposition'] = 'attachment; filename=%s' % filename + return response +export_to_csv.short_description = _(u"Export to CSV") + class PictureInline(admin.TabularInline): model = PictureFile extra = 1 @@ -102,7 +113,7 @@ class MarkerAdmin(admin.ModelAdmin): search_fields = ("name",) list_display = ('name', 'status') list_filter = ('status', 'categories') - actions = [validate, export_to_kml, export_to_shapefile] + actions = [validate, export_to_kml, export_to_shapefile, export_to_csv] exclude = ['submiter_session_key', 'import_key', 'import_version', 'available_date', 'ref_item'] readonly_fields = ['submiter_email', 'submiter_comment', 'import_source', @@ -199,7 +210,7 @@ class ImporterAdmin(admin.ModelAdmin): list_display = ('importer_type', 'source', 'state', 'filtr') list_filter = ('importer_type',) readonly_fields = ('state',) - actions = [importing, cancel_import, cancel_export] + actions = [importing, cancel_import, export_to_osm, cancel_export] admin.site.register(Importer, ImporterAdmin) class PageAdmin(admin.ModelAdmin): diff --git a/chimere/forms.py b/chimere/forms.py index 7315775..5cf2c63 100644 --- a/chimere/forms.py +++ b/chimere/forms.py @@ -137,7 +137,7 @@ class MarkerAdminFormBase(forms.ModelForm): """ Main form for marker """ - description = forms.CharField(widget=TextareaWidget, required=None) + description = forms.CharField(widget=TextareaWidget, required=False) class Meta: model = Marker diff --git a/chimere/migrations/0029_auto__add_field_marker_modified_since_import__add_field_marker_not_for.py b/chimere/migrations/0029_auto__add_field_marker_modified_since_import__add_field_marker_not_for.py new file mode 100644 index 0000000..72d718c --- /dev/null +++ b/chimere/migrations/0029_auto__add_field_marker_modified_since_import__add_field_marker_not_for.py @@ -0,0 +1,249 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'Marker.modified_since_import' + db.add_column('chimere_marker', 'modified_since_import', + self.gf('django.db.models.fields.BooleanField')(default=False), + keep_default=False) + + # Adding field 'Marker.not_for_osm' + db.add_column('chimere_marker', 'not_for_osm', + self.gf('django.db.models.fields.BooleanField')(default=False), + keep_default=False) + + # Adding field 'Route.modified_since_import' + db.add_column('chimere_route', 'modified_since_import', + self.gf('django.db.models.fields.BooleanField')(default=False), + keep_default=False) + + # Adding field 'Route.not_for_osm' + db.add_column('chimere_route', 'not_for_osm', + self.gf('django.db.models.fields.BooleanField')(default=False), + keep_default=False) + + + def backwards(self, orm): + # Deleting field 'Marker.modified_since_import' + db.delete_column('chimere_marker', 'modified_since_import') + + # Deleting field 'Marker.not_for_osm' + db.delete_column('chimere_marker', 'not_for_osm') + + # Deleting field 'Route.modified_since_import' + db.delete_column('chimere_route', 'modified_since_import') + + # Deleting field 'Route.not_for_osm' + db.delete_column('chimere_route', 'not_for_osm') + + + models = { + 'chimere.area': { + 'Meta': {'ordering': "('order', 'name')", 'object_name': 'Area'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'default': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'default_subcategories': ('chimere.widgets.SelectMultipleField', [], {'to': "orm['chimere.SubCategory']", 'symmetrical': 'False', 'blank': 'True'}), + 'dynamic_categories': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'external_css': ('django.db.models.fields.URLField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'layers': ('chimere.widgets.SelectMultipleField', [], {'symmetrical': 'False', 'related_name': "'areas'", 'blank': 'True', 'through': "orm['chimere.AreaLayers']", 'to': "orm['chimere.Layer']"}), + 'lower_right_corner': ('django.contrib.gis.db.models.fields.PointField', [], {'default': "'POINT(0 0)'"}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {}), + 'restrict_to_extent': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'subcategories': ('chimere.widgets.SelectMultipleField', [], {'symmetrical': 'False', 'related_name': "'areas'", 'blank': 'True', 'db_table': "'chimere_subcategory_areas'", 'to': "orm['chimere.SubCategory']"}), + 'upper_left_corner': ('django.contrib.gis.db.models.fields.PointField', [], {'default': "'POINT(0 0)'"}), + 'urn': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '50', 'blank': 'True'}), + 'welcome_message': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, + 'chimere.arealayers': { + 'Meta': {'ordering': "('order',)", 'object_name': 'AreaLayers'}, + 'area': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Area']"}), + 'default': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'layer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Layer']"}), + 'order': ('django.db.models.fields.IntegerField', [], {}) + }, + 'chimere.category': { + 'Meta': {'ordering': "['order']", 'object_name': 'Category'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {}) + }, + 'chimere.color': { + 'Meta': {'ordering': "['order']", 'object_name': 'Color'}, + 'code': ('django.db.models.fields.CharField', [], {'max_length': '6'}), + 'color_theme': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.ColorTheme']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'order': ('django.db.models.fields.IntegerField', [], {}) + }, + 'chimere.colortheme': { + 'Meta': {'object_name': 'ColorTheme'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}) + }, + 'chimere.icon': { + 'Meta': {'object_name': 'Icon'}, + 'height': ('django.db.models.fields.IntegerField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'image': ('django.db.models.fields.files.ImageField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'width': ('django.db.models.fields.IntegerField', [], {}) + }, + 'chimere.importer': { + 'Meta': {'object_name': 'Importer'}, + 'categories': ('chimere.widgets.SelectMultipleField', [], {'to': "orm['chimere.SubCategory']", 'symmetrical': 'False'}), + 'filtr': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'importer_type': ('django.db.models.fields.CharField', [], {'max_length': '4'}), + 'source': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'srid': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'state': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'zipped': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + }, + 'chimere.layer': { + 'Meta': {'object_name': 'Layer'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'layer_code': ('django.db.models.fields.TextField', [], {'max_length': '300'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}) + }, + 'chimere.marker': { + 'Meta': {'ordering': "('status', 'name')", 'object_name': 'Marker'}, + 'available_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'categories': ('chimere.widgets.SelectMultipleField', [], {'to': "orm['chimere.SubCategory']", 'symmetrical': 'False'}), + 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'import_key': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'import_source': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'import_version': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'modified_since_import': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'not_for_osm': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'point': ('chimere.widgets.PointField', [], {}), + 'ref_item': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'submited_marker'", 'null': 'True', 'to': "orm['chimere.Marker']"}), + 'route': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'associated_marker'", 'null': 'True', 'to': "orm['chimere.Route']"}), + 'start_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'submiter_comment': ('django.db.models.fields.TextField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'submiter_email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'submiter_name': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}), + 'submiter_session_key': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}) + }, + 'chimere.multimediafile': { + 'Meta': {'object_name': 'MultimediaFile'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'marker': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'multimedia_files'", 'to': "orm['chimere.Marker']"}), + 'miniature': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'multimedia_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.MultimediaType']"}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {'default': '1'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '200'}) + }, + 'chimere.multimediatype': { + 'Meta': {'object_name': 'MultimediaType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'iframe': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'media_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'mime_type': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}) + }, + 'chimere.news': { + 'Meta': {'object_name': 'News'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'content': ('django.db.models.fields.TextField', [], {}), + 'date': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '150'}) + }, + 'chimere.picturefile': { + 'Meta': {'object_name': 'PictureFile'}, + 'height': ('django.db.models.fields.IntegerField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'marker': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'pictures'", 'to': "orm['chimere.Marker']"}), + 'miniature': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {'default': '1'}), + 'picture': ('django.db.models.fields.files.ImageField', [], {'max_length': '100'}), + 'thumbnailfile': ('django.db.models.fields.files.ImageField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'thumbnailfile_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'thumbnailfile_width': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'width': ('django.db.models.fields.IntegerField', [], {}) + }, + 'chimere.property': { + 'Meta': {'object_name': 'Property'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'marker': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Marker']"}), + 'propertymodel': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.PropertyModel']"}), + 'value': ('django.db.models.fields.TextField', [], {}) + }, + 'chimere.propertymodel': { + 'Meta': {'ordering': "('order',)", 'object_name': 'PropertyModel'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mandatory': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {}), + 'subcategories': ('chimere.widgets.SelectMultipleField', [], {'symmetrical': 'False', 'related_name': "'properties'", 'blank': 'True', 'to': "orm['chimere.SubCategory']"}), + 'type': ('django.db.models.fields.CharField', [], {'max_length': '1'}) + }, + 'chimere.route': { + 'Meta': {'ordering': "('status', 'name')", 'object_name': 'Route'}, + 'associated_file': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.RouteFile']", 'null': 'True', 'blank': 'True'}), + 'categories': ('chimere.widgets.SelectMultipleField', [], {'to': "orm['chimere.SubCategory']", 'symmetrical': 'False'}), + 'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'import_key': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'import_source': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'import_version': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'modified_since_import': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'not_for_osm': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'picture': ('django.db.models.fields.files.ImageField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'ref_item': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'submited_route'", 'null': 'True', 'to': "orm['chimere.Route']"}), + 'route': ('chimere.widgets.RouteField', [], {}), + 'start_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'submiter_comment': ('django.db.models.fields.TextField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'submiter_email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'submiter_name': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}), + 'submiter_session_key': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}), + 'width': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}) + }, + 'chimere.routefile': { + 'Meta': {'ordering': "('name',)", 'object_name': 'RouteFile'}, + 'file_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'raw_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}), + 'simplified_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}) + }, + 'chimere.subcategory': { + 'Meta': {'ordering': "['category', 'order']", 'object_name': 'SubCategory'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'category': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Category']"}), + 'color_theme': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.ColorTheme']", 'null': 'True', 'blank': 'True'}), + 'icon': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Icon']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'item_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {}) + }, + 'chimere.tinyurl': { + 'Meta': {'object_name': 'TinyUrl'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'parameters': ('django.db.models.fields.CharField', [], {'max_length': '500'}) + } + } + + complete_apps = ['chimere']
\ No newline at end of file diff --git a/chimere/models.py b/chimere/models.py index d51cd5b..123b2fa 100644 --- a/chimere/models.py +++ b/chimere/models.py @@ -34,7 +34,7 @@ from django.contrib import admin from django.core.files import File from django.core.exceptions import ValidationError from django.core.urlresolvers import reverse -from django.db.models.signals import post_save +from django.db.models.signals import post_save, pre_save from django import forms from django.template import defaultfilters from django.utils.translation import ugettext_lazy as _ @@ -330,6 +330,10 @@ class GeographicItem(models.Model): blank=True, null=True) import_source = models.CharField(_(u"Source"), max_length=200, blank=True, null=True) + modified_since_import = models.BooleanField(_(u"Modified since last import"), + default=False) + not_for_osm = models.BooleanField(_(u"Not to be imported inside OSM"), + default=False) if settings.CHIMERE_DAYS_BEFORE_EVENT: start_date = models.DateField(_(u"Start date"), blank=True, null=True, help_text=_(u"Not mandatory. Set it for dated item such as event. "\ @@ -363,6 +367,10 @@ class GeographicItem(models.Model): self.import_key = new_keys self.save() + @classmethod + def properties(cls): + return [pm for pm in PropertyModel.objects.filter(available=True)] + class Marker(GeographicItem): '''Marker for a POI ''' @@ -386,8 +394,7 @@ class Marker(GeographicItem): super(Marker, self).__init__(*args, **kwargs) # add read attributes for properties for property in self.getProperties(): - attr_name = defaultfilters.slugify(property.propertymodel.name) - attr_name = re.sub(r'-','_', attr_name) + attr_name = property.propertymodel.getAttrName() if not hasattr(self, attr_name): setattr(self, attr_name, property.python_value) @@ -521,6 +528,33 @@ class Marker(GeographicItem): url = reverse('chimere:tiny', args=[area_name, urn]) return url +pre_save_marker_values = {} +def marker_pre_save(sender, **kwargs): + if not kwargs['instance']: + return + instance = kwargs['instance'] + pre_save_marker_values[instance.pk] = (instance.name, instance.point, + instance.import_version) +pre_save.connect(marker_pre_save, sender=Marker) + +def geometry_post_save(pre_save_geom_values): + def geom_post_save(sender, **kwargs): + if not kwargs['instance']: + return + instance = kwargs['instance'] + name, geometry, import_version = pre_save_geom_values[instance] + if (instance.import_version != import_version + and instance.modified_since_import): + instance.modified_since_import = False + instance.save() + return + if instance.modified_since_import: + return + if instance.name != name or instance.geometry != geometry: + instance.modified_since_import = True + return geom_post_save +post_save.connect(geometry_post_save(pre_save_marker_values), sender=Marker) + class MultimediaType(models.Model): MEDIA_TYPES = (('A', _(u"Audio")), ('V', _(u"Video")), @@ -874,6 +908,16 @@ class Route(GeographicItem): self.categories[0].id) return TinyUrl.getUrnByParameters(parameters) +pre_save_route_values = {} +def route_pre_save(sender, **kwargs): + if not kwargs['instance']: + return + instance = kwargs['instance'] + pre_save_route_values[instance.pk] = (instance.name, instance.route, + instance.import_version) +pre_save.connect(route_pre_save, sender=Route) +post_save.connect(geometry_post_save(pre_save_route_values), sender=Route) + def getDateCondition(): ''' Return an SQL condition for apparition of dates @@ -1105,6 +1149,11 @@ class PropertyModel(models.Model): ordering = ('order',) verbose_name = _("Property model") + def getAttrName(self): + attr_name = defaultfilters.slugify(self.name) + attr_name = re.sub(r'-','_', attr_name) + return attr_name + def getNamedId(self): '''Get the name used as named id (easily sortable) ''' diff --git a/chimere/tasks.py b/chimere/tasks.py index 1d55d2d..2d55bfd 100644 --- a/chimere/tasks.py +++ b/chimere/tasks.py @@ -22,6 +22,7 @@ if 'kombu.transport.django' in settings.INSTALLED_APPS \ and 'djcelery' in settings.INSTALLED_APPS: from celery.decorators import task +from django.core.cache import cache from django.core.exceptions import ObjectDoesNotExist from django.utils.translation import ugettext_lazy as _ @@ -44,7 +45,7 @@ if 'task' in globals(): release_lock = lambda: cache.delete(lock_id) if acquire_lock(): try: - func() + func(*args, **kwargs) finally: release_lock() return wrapper diff --git a/chimere/templates/chimere/export_marker.csv b/chimere/templates/chimere/export_marker.csv new file mode 100644 index 0000000..619b872 --- /dev/null +++ b/chimere/templates/chimere/export_marker.csv @@ -0,0 +1,2 @@ +{% for row in data %}{% for dat in row %}{% if forloop.counter0 %};{% endif %}"{{ dat|safe|addslashes }}"{% endfor %} +{% endfor %} diff --git a/chimere/utils.py b/chimere/utils.py index d61ad19..a24e22f 100644 --- a/chimere/utils.py +++ b/chimere/utils.py @@ -21,6 +21,7 @@ Utilitaries """ +import csv import datetime import os import re @@ -429,9 +430,161 @@ class ShapefileManager(ImportManager): buff.close() return filename, zip_stream -RE_NODE = re.compile('node\[([^\]]*)\]') +class ImportManager: + u""" + Generic class for specific importers + """ + default_source = None + def __init__(self, importer_instance): + self.importer_instance = importer_instance + self.default_name = " - ".join([cat.name + for cat in self.importer_instance.categories.order_by('name').all()]) + + def get(self): + pass + + def put(self): + pass + + def create_or_update_item(self, cls, values, import_key, version=None): + updated, created, item = False, False, None + if import_key: + dct_import = { + 'import_key__icontains':'%s:%s;' % ( + self.importer_instance.importer_type, + import_key), + 'import_source':self.importer_instance.source} + try: + item = cls.objects.get(**dct_import) + if version and item.import_version == int(version): + # no update since the last import + return item, None, None + for k in values: + setattr(item, k, values[k]) + try: + item.save() + except TypeError: + # error on data source + return None, False, False + updated = True + except ObjectDoesNotExist: + pass + if not item: + values.update({ + 'import_source':self.importer_instance.source}) + values['status'] = 'I' + try: + item = cls.objects.create(**values) + except TypeError: + # error on data source + return None, False, False + created = True + if import_key: + item.set_key(self.importer_instance.importer_type, + import_key) + item.categories.clear() + for cat in self.importer_instance.categories.all(): + item.categories.add(cat) + return item, updated, created + + @classmethod + def get_files_inside_zip(cls, zippedfile, suffixes, dest_dir=None): + try: + flz = zipfile.ZipFile(zippedfile) + except zipfile.BadZipfile: + return [], _(u"Bad zip file") + namelist = flz.namelist() + filenames = [] + for suffix in suffixes: + current_file_name = None + for name in namelist: + if name.endswith(suffix) \ + or name.endswith(suffix.lower()) \ + or name.endswith(suffix.upper()): + current_file_name = name + filenames.append(current_file_name) + files = [] + for filename in filenames: + if filename: + if dest_dir: + files.append(filename) + flz.extract(filename, dest_dir) + else: + files.append(flz.open(filename)) + else: + files.append(None) + return files -# manage deleted item from OSM + def get_source_file(self, source, suffixes, dest_dir=None, + extra_url=None): + if not hasattr(source, 'read'): + if not source: + source = self.importer_instance.source \ + if self.importer_instance.source else self.default_source + try: + url = source + if extra_url: + url += extra_url + remotehandle = urllib2.urlopen(url) + source = StringIO.StringIO(remotehandle.read()) + remotehandle.close() + except ValueError: + # assume it is a local file + try: + source = open(source) + except IOError, msg: + return (None, msg) + except urllib2.URLError as error: + return (None, error.message) + if self.importer_instance.zipped: + try: + files = self.get_files_inside_zip(source, suffixes, dest_dir) + except zipfile.BadZipfile: + return (None, _(u"Bad zip file")) + if not files or None in files: + return (None, + _(u"Missing file(s) inside the zip file")) + source = files[0] if len(suffixes) == 1 else files + return (source, None) + +class CSVManager(ImportManager): + u""" + CSV importer + """ + _COLS = [("Id", 'pk'), (_(u"Name"), 'name'), + (_(u"Categories"), lambda obj:", ".join( + [c.name for c in obj.categories.all()])), + (_(u"State"), 'status')] + COLS = {'marker':_COLS+[(_(u"Description"), 'description'), + (_(u"Localisation"), lambda obj: obj.point.wkt)], + 'route':_COLS+[(_(u"Localisation"), lambda obj: obj.route.wkt)]} + + @classmethod + def export(cls, queryset): + dct = {'description':unicode(datetime.date.today()), 'data':[]} + cls_name = queryset.model.__name__.lower() + cols = cls.COLS[cls_name][:] + for pm in queryset.model.properties(): + cols.append((pm.name, pm.getAttrName())) + header = [lbl for lbl, attr in cols] + dct['data'].append(header) + for item in queryset.all(): + data = [] + for lbl, attr in cols: + if callable(attr): + data.append(attr(item)) + print lbl, attr(item) + else: + data.append(getattr(item, attr)) + dct['data'].append(data) + filename = unicode_normalize(settings.PROJECT_NAME + dct['description']\ + + '.csv') + result = render_to_response('chimere/export_%s.csv' % cls_name, dct) + return filename, result + +RE_HOOK = re.compile('\[([^\]]*)\]') + +# TODO: manage deleted item from OSM class OSMManager(ImportManager): u""" @@ -554,26 +707,58 @@ class OSMManager(ImportManager): password=settings.CHIMERE_OSM_PASSWORD) api.ChangesetCreate({u"comment": u"Import from Chimère %s" % \ get_version()}) - tag = RE_NODE.finddall(self.importer_instance.filtr) - if not tag: + hooks = RE_HOOK.findall(self.importer_instance.filtr) + if not hooks: return 0, _(u"Bad param") - tag = tag[0].split('=') - default_dct = {'tag':{tag[0]:tag[1]}, + tags = {} + bbox = [] + for hook in hooks: + key, value = hook.split('=') + if '*' in value or '|' in key or '|' in value: + continue + if key == 'bbox': + x1, y1, x2, y2 = [float(val) for val in value.split(',')] + bbox = GEOSGeometry( + 'POLYGON((%f %f,%f %f,%f %f,%f %f,%f %f))' % ( + x1, y1, x2, y1, x2, y2, x1, y2, x1, y1), srid=4326) + continue + tags[key] = value + if not tags: + return 0, _(u"No non ambigious tag is defined in the XAPI request") + if not bbox: + return 0, _(u"No bounding box is defined in the XAPI request."\ + u"If you are sure to manage the entire planet set the bounding box"\ + u" to -180,-90,180,90") + default_dct = {'tag':tags, 'import_source':self.importer_instance.source} - for idx, item in Marker.objects.filter(status='A', - categories=self.importer_instance.categories.all()): - dct = default_dct.update({ - 'name':item.name, - 'lon':item.point.lon, - 'lat':item.point.lat}) + idx = -1 + for idx, item in enumerate(Marker.objects.filter(status='A', + point__contained=bbox, + categories=self.importer_instance.categories.all(), + not_for_osm=False, modified_since_import=True).all()): + dct = default_dct.copy() + dct.update({'lon':item.point.x, + 'lat':item.point.y}) + dct['tag']['name'] = item.name node = None - import_key = marker.get_key('OSM') - if not import_key: - node = OsmApi.NodeCreate(dct) + import_key = item.get_key('OSM') + updated = False + if import_key: + try: + dct['id'] = import_key + dct['version'] = item.import_version + node = api.NodeUpdate(dct) + updated = True + except OsmApi.ApiError, error: + if error.status == 404: + dct.pop('id') + dct.pop('version') + pass # if the node doesn't exist it is created + else: + raise + if not updated: + node = api.NodeCreate(dct) item.set_key('OSM', node['id']) - else: - dct['id'] = import_key - node = OsmApi.NodeUpdate(dct) item.import_version = node['version'] item.save() api.ChangesetClose() diff --git a/example_project/settings.py b/example_project/settings.py index f67ceeb..372f1e6 100644 --- a/example_project/settings.py +++ b/example_project/settings.py @@ -105,6 +105,7 @@ CHIMERE_THUMBS_SCALE_WIDTH=None CHIMERE_SHORT_DESC_LENGTH = 400 CHIMERE_MODIF_EMAIL = _(u"""Hello, I would like to propose you a modification about this item: """) +CHIMERE_CSV_ENCODING = 'ISO-8859-1' ADMINS = ( # ('Your Name', 'your_email@domain.com'), |