From f8c2ecf2fe3aa78c64b2fa3b29979fe55e5636cf Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Thu, 21 May 2015 18:36:19 +0200 Subject: Fix modified_since_import in imports --- chimere/utils.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'chimere/utils.py') diff --git a/chimere/utils.py b/chimere/utils.py index 55fc45c..5b34e9c 100644 --- a/chimere/utils.py +++ b/chimere/utils.py @@ -131,6 +131,8 @@ class ImportManager(object): values['has_associated_marker'] = False try: item = cls.objects.create(**values) + item.modified_since_import = False + item.save() except TypeError: # error on data source return None, False, False -- cgit v1.2.3 From 5fe6d580e85dc7208f48c89e417290cee7ec9d33 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Thu, 21 May 2015 18:41:20 +0200 Subject: Do not re-import modified since import --- chimere/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'chimere/utils.py') diff --git a/chimere/utils.py b/chimere/utils.py index 5b34e9c..a765c32 100644 --- a/chimere/utils.py +++ b/chimere/utils.py @@ -100,7 +100,7 @@ class ImportManager(object): return ref_item, None, None if not self.importer_instance.overwrite \ and ref_item.modified_since_import: - dct_import['ref_item'] = ref_item + return ref_item, None, None else: item = ref_item for k in values: -- cgit v1.2.3 From 7f62e7d756265c88556d0afd8aa9ae6c941dfe5f Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Fri, 22 May 2015 20:39:37 +0200 Subject: HTML/XML imports: origin is a link with target blank --- chimere/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'chimere/utils.py') diff --git a/chimere/utils.py b/chimere/utils.py index a765c32..2fe5fb4 100644 --- a/chimere/utils.py +++ b/chimere/utils.py @@ -1065,7 +1065,7 @@ class HtmlXsltManager(ImportManager): not "point" in item and not ("lat" in item and item['lat']): return cls = None - dct = {'origin':"%s" % (item['link'], + dct = {'origin':"%s" % (item['link'], self.importer_instance.origin), 'license':self.importer_instance.license, 'name':item['name']} -- cgit v1.2.3 From bfeab1a4d50e6d8a941a9de7b034d86d7704926d Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Tue, 28 Jul 2015 21:02:52 +0200 Subject: XSLT import : default link --- chimere/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'chimere/utils.py') diff --git a/chimere/utils.py b/chimere/utils.py index 2fe5fb4..d898759 100644 --- a/chimere/utils.py +++ b/chimere/utils.py @@ -1065,7 +1065,8 @@ class HtmlXsltManager(ImportManager): not "point" in item and not ("lat" in item and item['lat']): return cls = None - dct = {'origin':"%s" % (item['link'], + dct = {'origin':"%s" % ( + item.get('link') or '#', self.importer_instance.origin), 'license':self.importer_instance.license, 'name':item['name']} -- cgit v1.2.3 From 5d2cac2cb5a03c8f55a2396e4c4ce13911a9149c Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Wed, 23 Sep 2015 23:33:10 +0200 Subject: Imports: specify the status used by default during imports --- ...0009_auto__add_field_importer_default_status.py | 289 +++++++++++++++++++++ chimere/models.py | 16 +- chimere/static/chimere/js/importer_interface.js | 16 +- chimere/utils.py | 3 +- 4 files changed, 308 insertions(+), 16 deletions(-) create mode 100644 chimere/migrations/0009_auto__add_field_importer_default_status.py (limited to 'chimere/utils.py') diff --git a/chimere/migrations/0009_auto__add_field_importer_default_status.py b/chimere/migrations/0009_auto__add_field_importer_default_status.py new file mode 100644 index 0000000..e29c599 --- /dev/null +++ b/chimere/migrations/0009_auto__add_field_importer_default_status.py @@ -0,0 +1,289 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'Importer.default_status' + db.add_column('chimere_importer', 'default_status', + self.gf('django.db.models.fields.CharField')(default='I', max_length=1), + keep_default=False) + + + def backwards(self, orm): + # Deleting field 'Importer.default_status' + db.delete_column('chimere_importer', 'default_status') + + + models = { + 'chimere.aggregatedroute': { + 'Meta': {'object_name': 'AggregatedRoute', 'db_table': "'chimere_aggregated_routes'", 'managed': 'False'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'route': ('django.contrib.gis.db.models.fields.MultiLineStringField', [], {}), + 'status': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'subcategory': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.SubCategory']"}) + }, + 'chimere.area': { + 'Meta': {'ordering': "('order', 'name')", 'object_name': 'Area'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'default': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'default_subcategories': ('chimere.widgets.SelectMultipleField', [], {'to': "orm['chimere.SubCategory']", 'symmetrical': 'False', 'blank': 'True'}), + 'dynamic_categories': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'external_css': ('django.db.models.fields.URLField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'layers': ('chimere.widgets.SelectMultipleField', [], {'symmetrical': 'False', 'related_name': "'areas'", 'blank': 'True', 'through': "orm['chimere.AreaLayers']", 'to': "orm['chimere.Layer']"}), + 'lower_right_corner': ('django.contrib.gis.db.models.fields.PointField', [], {'default': "'POINT(0 0)'"}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {'unique': 'True'}), + 'restrict_to_extent': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'subcategories': ('chimere.widgets.SelectMultipleField', [], {'symmetrical': 'False', 'related_name': "'areas'", 'blank': 'True', 'db_table': "'chimere_subcategory_areas'", 'to': "orm['chimere.SubCategory']"}), + 'upper_left_corner': ('django.contrib.gis.db.models.fields.PointField', [], {'default': "'POINT(0 0)'"}), + 'urn': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '50', 'blank': 'True'}), + 'welcome_message': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, + 'chimere.arealayers': { + 'Meta': {'ordering': "('order',)", 'object_name': 'AreaLayers'}, + 'area': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Area']"}), + 'default': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'layer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Layer']"}), + 'order': ('django.db.models.fields.IntegerField', [], {}) + }, + 'chimere.category': { + 'Meta': {'ordering': "['order']", 'object_name': 'Category'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {}) + }, + 'chimere.color': { + 'Meta': {'ordering': "['order']", 'object_name': 'Color'}, + 'code': ('django.db.models.fields.CharField', [], {'max_length': '6'}), + 'color_theme': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.ColorTheme']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'order': ('django.db.models.fields.IntegerField', [], {}) + }, + 'chimere.colortheme': { + 'Meta': {'object_name': 'ColorTheme'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}) + }, + 'chimere.icon': { + 'Meta': {'object_name': 'Icon'}, + 'height': ('django.db.models.fields.IntegerField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'image': ('django.db.models.fields.files.ImageField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'width': ('django.db.models.fields.IntegerField', [], {}) + }, + 'chimere.importer': { + 'Meta': {'object_name': 'Importer'}, + 'associate_marker_to_way': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'automatic_update': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'categories': ('chimere.widgets.SelectMultipleField', [], {'symmetrical': 'False', 'to': "orm['chimere.SubCategory']", 'null': 'True', 'blank': 'True'}), + 'default_description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'default_localisation': ('chimere.widgets.PointField', [], {'null': 'True', 'blank': 'True'}), + 'default_name': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'default_status': ('django.db.models.fields.CharField', [], {'default': "'I'", 'max_length': '1'}), + 'filtr': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'get_description': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'importer_type': ('django.db.models.fields.CharField', [], {'max_length': '4'}), + 'license': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'origin': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'overwrite': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'source': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'source_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'source_file_alt': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'srid': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'state': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'zipped': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + }, + 'chimere.importerkeycategories': { + 'Meta': {'object_name': 'ImporterKeyCategories'}, + 'category': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.SubCategory']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'importer': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'key_categories'", 'to': "orm['chimere.Importer']"}), + 'key': ('django.db.models.fields.CharField', [], {'max_length': '200'}) + }, + 'chimere.layer': { + 'Meta': {'object_name': 'Layer'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'layer_code': ('django.db.models.fields.TextField', [], {'max_length': '300'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}) + }, + 'chimere.marker': { + 'Meta': {'ordering': "('status', 'name')", 'object_name': 'Marker'}, + 'available_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'categories': ('chimere.widgets.SelectMultipleField', [], {'to': "orm['chimere.SubCategory']", 'symmetrical': 'False'}), + 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'import_key': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'import_source': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'import_version': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'is_front_page': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'license': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'modified_since_import': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'not_for_osm': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'origin': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'point': ('chimere.widgets.PointField', [], {}), + 'ref_item': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'submited_marker'", 'null': 'True', 'to': "orm['chimere.Marker']"}), + 'route': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'associated_marker'", 'null': 'True', 'to': "orm['chimere.Route']"}), + 'start_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'submiter_comment': ('django.db.models.fields.TextField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'submiter_email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'submiter_name': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}), + 'submiter_session_key': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}) + }, + 'chimere.multimediaextension': { + 'Meta': {'object_name': 'MultimediaExtension'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'multimedia_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'extensions'", 'to': "orm['chimere.MultimediaType']"}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '6'}) + }, + 'chimere.multimediafile': { + 'Meta': {'object_name': 'MultimediaFile'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'marker': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'multimedia_files'", 'to': "orm['chimere.Marker']"}), + 'miniature': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'multimedia_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.MultimediaType']", 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {'default': '1'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '200'}) + }, + 'chimere.multimediatype': { + 'Meta': {'object_name': 'MultimediaType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'iframe': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'media_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'mime_type': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}) + }, + 'chimere.news': { + 'Meta': {'object_name': 'News'}, + 'areas': ('chimere.widgets.SelectMultipleField', [], {'symmetrical': 'False', 'to': "orm['chimere.Area']", 'null': 'True', 'blank': 'True'}), + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'content': ('django.db.models.fields.TextField', [], {}), + 'date': ('django.db.models.fields.DateField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_front_page': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}) + }, + 'chimere.page': { + 'Meta': {'object_name': 'Page'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'content': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mnemonic': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}), + 'order': ('django.db.models.fields.IntegerField', [], {'default': '10', 'null': 'True', 'blank': 'True'}), + 'template_path': ('django.db.models.fields.CharField', [], {'max_length': '150', 'null': 'True', 'blank': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '150'}) + }, + 'chimere.picturefile': { + 'Meta': {'object_name': 'PictureFile'}, + 'height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'marker': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'pictures'", 'to': "orm['chimere.Marker']"}), + 'miniature': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {'default': '1'}), + 'picture': ('django.db.models.fields.files.ImageField', [], {'max_length': '100'}), + 'thumbnailfile': ('django.db.models.fields.files.ImageField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'thumbnailfile_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'thumbnailfile_width': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'width': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}) + }, + 'chimere.property': { + 'Meta': {'object_name': 'Property'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'marker': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Marker']"}), + 'propertymodel': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.PropertyModel']"}), + 'value': ('django.db.models.fields.TextField', [], {}) + }, + 'chimere.propertymodel': { + 'Meta': {'ordering': "('order',)", 'object_name': 'PropertyModel'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mandatory': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {}), + 'subcategories': ('chimere.widgets.SelectMultipleField', [], {'symmetrical': 'False', 'related_name': "'properties'", 'blank': 'True', 'to': "orm['chimere.SubCategory']"}), + 'type': ('django.db.models.fields.CharField', [], {'max_length': '1'}) + }, + 'chimere.propertymodelchoice': { + 'Meta': {'object_name': 'PropertyModelChoice'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'propertymodel': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'choices'", 'to': "orm['chimere.PropertyModel']"}), + 'value': ('django.db.models.fields.CharField', [], {'max_length': '150'}) + }, + 'chimere.route': { + 'Meta': {'ordering': "('status', 'name')", 'object_name': 'Route'}, + 'associated_file': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.RouteFile']", 'null': 'True', 'blank': 'True'}), + 'categories': ('chimere.widgets.SelectMultipleField', [], {'to': "orm['chimere.SubCategory']", 'symmetrical': 'False'}), + 'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'has_associated_marker': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'import_key': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'import_source': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'import_version': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'license': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'modified_since_import': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'not_for_osm': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'origin': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'picture': ('django.db.models.fields.files.ImageField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'ref_item': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'submited_route'", 'null': 'True', 'to': "orm['chimere.Route']"}), + 'route': ('chimere.widgets.RouteField', [], {}), + 'start_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'submiter_comment': ('django.db.models.fields.TextField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'submiter_email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'submiter_name': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}), + 'submiter_session_key': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}), + 'width': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}) + }, + 'chimere.routefile': { + 'Meta': {'ordering': "('name',)", 'object_name': 'RouteFile'}, + 'file_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'raw_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}), + 'simplified_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}) + }, + 'chimere.subcategory': { + 'Meta': {'ordering': "['category', 'order']", 'object_name': 'SubCategory'}, + 'as_layer': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'category': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'subcategories'", 'to': "orm['chimere.Category']"}), + 'color_theme': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.ColorTheme']", 'null': 'True', 'blank': 'True'}), + 'dated': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'hover_icon': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'subcat_hovered'", 'null': 'True', 'to': "orm['chimere.Icon']"}), + 'icon': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Icon']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'item_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {'default': '1000'}), + 'routing_warn': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'submission': ('django.db.models.fields.BooleanField', [], {'default': 'True'}) + }, + 'chimere.tinyurl': { + 'Meta': {'object_name': 'TinyUrl'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'parameters': ('django.db.models.fields.CharField', [], {'max_length': '500'}) + } + } + + complete_apps = ['chimere'] \ No newline at end of file diff --git a/chimere/models.py b/chimere/models.py index 2f30892..757e464 100644 --- a/chimere/models.py +++ b/chimere/models.py @@ -299,6 +299,13 @@ class SubCategory(models.Model): json_string = json.dumps(self.getJSONDict()) return json_string +STATUS = (('S', _(u'Submited')), + ('A', _(u'Available')), + ('M', _(u'Modified')), + ('D', _(u'Disabled')), + ('I', _(u'Imported'))) +STATUS_DCT = dict(STATUS) + IMPORTERS = {'KML':KMLManager, 'OSM':OSMManager, 'SHP':ShapefileManager, @@ -355,6 +362,8 @@ class Importer(models.Model): u"a marker to a way"), default=False) automatic_update = models.BooleanField(_(u"Automatically updated"), default=False) + default_status = models.CharField(_(u"Default status"), max_length=1, + choices=STATUS, default='I') default_localisation = PointField(_(u"Default localisation"), srid=settings.CHIMERE_EPSG_DISPLAY_PROJECTION, blank=True, null=True, @@ -403,13 +412,6 @@ class ImporterKeyCategories(models.Model): class Meta: verbose_name = _(u"Importer - Key categories") -STATUS = (('S', _(u'Submited')), - ('A', _(u'Available')), - ('M', _(u'Modified')), - ('D', _(u'Disabled')), - ('I', _(u'Imported'))) -STATUS_DCT = dict(STATUS) - class GeographicItem(models.Model): name = models.CharField(_(u"Name"), max_length=150) categories = SelectMultipleField(SubCategory) diff --git a/chimere/static/chimere/js/importer_interface.js b/chimere/static/chimere/js/importer_interface.js index 4c6e29d..5c77a8d 100644 --- a/chimere/static/chimere/js/importer_interface.js +++ b/chimere/static/chimere/js/importer_interface.js @@ -2,35 +2,37 @@ django.jQuery(function($) { var importer_form_filter = { OSM:new Array('field-filtr', 'field-default_name', 'field-categories', 'field-source', 'field-overwrite', - 'field-automatic_update'), + 'field-automatic_update', 'field-default_status'), KML:new Array('field-source', 'field-source_file', 'field-default_name', 'field-filtr', 'field-zipped', 'field-origin', 'field-license', 'field-categories', 'field-overwrite', - 'field-get_description', 'field-automatic_update'), + 'field-get_description', 'field-automatic_update', + 'field-default_status'), SHP:new Array('field-source', 'field-source_file', 'field-default_name', 'field-zipped', 'field-origin', 'field-srid', 'field-license', 'field-categories', 'field-overwrite', - 'field-automatic_update'), + 'field-automatic_update', 'field-default_status'), RSS:new Array('field-source', 'field-default_name', 'field-origin', 'field-srid', 'field-license', 'field-categories', 'field-overwrite', 'field-get_description', - 'field-automatic_update'), + 'field-automatic_update', 'field-default_status'), CSV:new Array('field-source', 'field-source_file', 'field-default_name', 'field-origin', 'field-srid', 'field-license', 'field-categories', 'field-overwrite', - 'field-get_description', 'field-automatic_update'), + 'field-get_description', 'field-automatic_update', + 'field-default_status'), XSLT:new Array('field-source', 'field-source_file', 'field-source_file_alt', 'field-default_name', 'field-origin', 'field-srid', 'field-license', 'field-categories', 'field-overwrite', 'field-get_description', 'field-default_localisation', - 'field-automatic_update'), + 'field-automatic_update', 'field-default_status'), XXLT:new Array('field-source', 'field-source_file', 'field-source_file_alt', 'field-default_name', 'field-origin', 'field-srid', 'field-license', 'field-categories', 'field-overwrite', 'field-get_description', 'field-default_localisation', - 'field-automatic_update') + 'field-automatic_update', 'field-default_status') } var osm_map_initialized; var edit_map_initialized; diff --git a/chimere/utils.py b/chimere/utils.py index d898759..c5c59e9 100644 --- a/chimere/utils.py +++ b/chimere/utils.py @@ -124,8 +124,7 @@ class ImportManager(object): self.importer_instance.default_description values.update({ 'import_source':self.importer_instance.source}) - values['status'] = 'I' \ - if not self.importer_instance.automatic_update else 'A' + values['status'] = self.importer_instance.default_status if not self.importer_instance.associate_marker_to_way\ and cls.__name__ == 'Route': values['has_associated_marker'] = False -- cgit v1.2.3 From d2ca6b5ea37832ba7477b3d02425b83ca0938f49 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Thu, 19 Nov 2015 16:39:40 +0100 Subject: Flake8 --- chimere/tasks.py | 37 +++--- chimere/utils.py | 370 +++++++++++++++++++++++++++++-------------------------- 2 files changed, 216 insertions(+), 191 deletions(-) (limited to 'chimere/utils.py') diff --git a/chimere/tasks.py b/chimere/tasks.py index 9c94f43..9eff7f5 100644 --- a/chimere/tasks.py +++ b/chimere/tasks.py @@ -60,19 +60,20 @@ else: return task_exc IMPORT_MESSAGES = { - 'import_pending':[_(u"Import pending")], - 'import_process':[_(u"Import processing")], - 'import_done':[_(u"Import successfuly done"), + 'import_pending': [_(u"Import pending")], + 'import_process': [_(u"Import processing")], + 'import_done': [_(u"Import successfuly done"), _(u" %(new)d new item(s), %(updated)d updated item(s)")], - 'import_failed':[_(u"Import failed"), "%s"], - 'import_cancel':[_(u"Import canceled")], - 'export_pending':[_(u"Export pending")], - 'export_process':[_(u"Export processing")], - 'export_done':[_(u"Export successfuly done"), + 'import_failed': [_(u"Import failed"), "%s"], + 'import_cancel': [_(u"Import canceled")], + 'export_pending': [_(u"Export pending")], + 'export_process': [_(u"Export processing")], + 'export_done': [_(u"Export successfuly done"), _(u" %(updated)d updated item(s)")], - 'export_failed':[_(u"Export failed"), "%s"], - 'export_cancel':[_(u"Export canceled")] - } + 'export_failed': [_(u"Export failed"), "%s"], + 'export_cancel': [_(u"Export canceled")] +} + @task() def importing(importer_pk): @@ -89,15 +90,16 @@ def importing(importer_pk): new_item, updated_item, error = importer.manager.get() importer.state = error + '\n' if error else '' importer.state += unicode(IMPORT_MESSAGES['import_done'][0]) - importer.state += u" - " \ - + unicode(IMPORT_MESSAGES['import_done'][1]) % {'new':new_item, - 'updated':updated_item} + importer.state += \ + u" - " + unicode(IMPORT_MESSAGES['import_done'][1]) % { + 'new': new_item, 'updated': updated_item} importer.state = importer.state importer.save() return True + @task() -@single_instance_task(60*10) +@single_instance_task(60 * 10) def exporting(importer_pk, extra_args=[]): try: importer = Importer.objects.get(pk=importer_pk) @@ -116,10 +118,11 @@ def exporting(importer_pk, extra_args=[]): pass if error: importer.state = unicode(IMPORT_MESSAGES['export_failed'][0]) \ - + u" - " + unicode(IMPORT_MESSAGES['export_failed'][1]) % error + + u" - " + unicode(IMPORT_MESSAGES['export_failed'][1]) % error importer.save() return importer.state = unicode(IMPORT_MESSAGES['export_done'][0]) + u" - " \ - + unicode(IMPORT_MESSAGES['export_done'][1]) % {'updated':updated_item} + + unicode(IMPORT_MESSAGES['export_done'][1]) % { + 'updated': updated_item} importer.save() return True diff --git a/chimere/utils.py b/chimere/utils.py index c5c59e9..e017762 100644 --- a/chimere/utils.py +++ b/chimere/utils.py @@ -45,26 +45,30 @@ from django.utils.translation import ugettext_lazy as _ from chimere import get_version from external_utils import OsmApi + def unicode_normalize(string): if type(string) == str: string = unicode(string.decode('utf-8')) return ''.join( (c for c in unicodedata.normalize('NFD', string) - if unicodedata.category(c) != 'Mn')) + if unicodedata.category(c) != 'Mn')) + class ImportManager(object): u""" Generic class for specific importers """ default_source = None + def __init__(self, importer_instance): self.importer_instance = importer_instance if self.importer_instance.default_name: self.default_name = self.importer_instance.default_name else: - self.default_name = " - ".join([cat.name + self.default_name = " - ".join([ + cat.name for cat in self.importer_instance.categories.order_by( - 'name').all()]) + 'name').all()]) def get(self): raise NotImplementedError @@ -83,8 +87,8 @@ class ImportManager(object): item = None if import_key or pk: dct_import = { - 'import_key__icontains':'%s:%s;' % (key, import_key), - 'import_source':self.importer_instance.source} + 'import_key__icontains': '%s:%s;' % (key, import_key), + 'import_source': self.importer_instance.source} ref_item = cls.objects.filter(**dct_import) try: item = None @@ -121,12 +125,12 @@ class ImportManager(object): if not self.importer_instance.get_description and \ self.importer_instance.default_description: values['description'] = \ - self.importer_instance.default_description + self.importer_instance.default_description values.update({ - 'import_source':self.importer_instance.source}) + 'import_source': self.importer_instance.source}) values['status'] = self.importer_instance.default_status if not self.importer_instance.associate_marker_to_way\ - and cls.__name__ == 'Route': + and cls.__name__ == 'Route': values['has_associated_marker'] = False try: item = cls.objects.create(**values) @@ -158,8 +162,8 @@ class ImportManager(object): current_file_name = None for name in namelist: if name.endswith(suffix) \ - or name.endswith(suffix.lower()) \ - or name.endswith(suffix.upper()): + or name.endswith(suffix.lower()) \ + or name.endswith(suffix.upper()): current_file_name = name filenames.append(current_file_name) files = [] @@ -180,7 +184,7 @@ class ImportManager(object): if not hasattr(source, 'read'): if not source: source = self.importer_instance.source \ - if self.importer_instance.source else self.default_source + if self.importer_instance.source else self.default_source try: url = source if extra_url: @@ -207,6 +211,7 @@ class ImportManager(object): source = files[0] if len(suffixes) == 1 else files return (source, None) + class KMLManager(ImportManager): u""" KML importer @@ -215,6 +220,7 @@ class KMLManager(ImportManager): """ XPATH = '//kml:Folder/kml:name[text()="%s"]/../kml:Placemark' DEFAULT_XPATH = '//kml:Placemark' + def __init__(self, importer_instance, ns=''): super(KMLManager, self).__init__(importer_instance) self.ns = ns @@ -249,9 +255,9 @@ class KMLManager(ImportManager): if not self.ns: self.ns = tree.getroot().nsmap[None] xpath = self.XPATH % self.importer_instance.filtr \ - if self.importer_instance.filtr else self.DEFAULT_XPATH + if self.importer_instance.filtr else self.DEFAULT_XPATH for placemark in tree.xpath(xpath, - namespaces={'kml':self.ns}): + namespaces={'kml': self.ns}): name, point, line = None, None, None pl_id = placemark.attrib.get('id') pl_key = 'kml-%d' % self.importer_instance.pk @@ -279,10 +285,10 @@ class KMLManager(ImportManager): for p in points if p]) line = 'SRID=4326;LINESTRING(%s)' % points cls = None - dct = {'description':description, - 'name':name, - 'origin':self.importer_instance.origin, - 'license':self.importer_instance.license} + dct = {'description': description, + 'name': name, + 'origin': self.importer_instance.origin, + 'license': self.importer_instance.license} if point: dct['point'] = point cls = Marker @@ -292,7 +298,7 @@ class KMLManager(ImportManager): cls = Route if cls: item, updated, created = self.create_or_update_item( - cls, dct, pl_id, key=pl_key) + cls, dct, pl_id, key=pl_key) if updated: updated_item += 1 if created: @@ -301,15 +307,17 @@ class KMLManager(ImportManager): @classmethod def export(cls, queryset): - dct = {'name':settings.PROJECT_NAME, - 'description':unicode(datetime.date.today()), - 'locations':queryset.all() + dct = { + 'name': settings.PROJECT_NAME, + 'description': unicode(datetime.date.today()), + 'locations': queryset.all() } - filename = unicode_normalize(settings.PROJECT_NAME + dct['description']\ + filename = unicode_normalize(settings.PROJECT_NAME + dct['description'] + '.kml') result = render_to_response('chimere/export.kml', dct) return filename, result + class ShapefileManager(ImportManager): u""" Shapefile importer @@ -351,7 +359,7 @@ class ShapefileManager(ImportManager): srid = settings.CHIMERE_EPSG_DISPLAY_PROJECTION msg = _(u"SRID cannot be guessed. The default SRID (%s) has " u"been used.") % srid - #If imported items are not well located " + # If imported items are not well located " # u"ask your data provider for the SRID to use.") % srid shapefilename = tmpdir + os.sep + sources[0] ds = DataSource(shapefilename) @@ -374,7 +382,7 @@ class ShapefileManager(ImportManager): u"is not managed by Chimère.") % lyr.geom_type) geom_key = 'point' if lyr.geom_type == 'Point' else 'route' geom_cls = Marker if lyr.geom_type == 'Point' else Route - indexes = [] + # indexes = [] for idx, feat in enumerate(lyr): name = unicode(idx) if lbl_name: @@ -384,7 +392,7 @@ class ShapefileManager(ImportManager): except UnicodeDecodeError: try: name = unicode( - name.decode(settings.CHIMERE_SHAPEFILE_ENCODING)) + name.decode(settings.CHIMERE_SHAPEFILE_ENCODING)) except: continue try: @@ -393,15 +401,17 @@ class ShapefileManager(ImportManager): return (0, 0, _(u"Bad Shapefile")) if feat.geom.geom_type == 'MultiLineString': geoms = [geom.wkt for geom in feat.geom] - import_key = feat.get(id_name) if id_name and len(geoms) == 1 else '' + import_key = feat.get(id_name) if id_name and len(geoms) == 1 \ + else '' for geom in geoms: - dct = {geom_key:'SRID=%s;%s' % (srid, geom), - 'name':name, - 'origin':self.importer_instance.origin, - 'license':self.importer_instance.license - } + dct = { + geom_key: 'SRID=%s;%s' % (srid, geom), + 'name': name, + 'origin': self.importer_instance.origin, + 'license': self.importer_instance.license + } item, updated, created = self.create_or_update_item( - geom_cls, dct, import_key) + geom_cls, dct, import_key) if updated: updated_item += 1 if created: @@ -426,8 +436,9 @@ class ShapefileManager(ImportManager): tmp_name = tmp.name field_names = [field.name for field in queryset.model._meta.fields] - geo_field = getattr(queryset.model, - 'point' if 'point' in field_names else 'route')._field + geo_field = getattr( + queryset.model, + 'point' if 'point' in field_names else 'route')._field dr = ogr.GetDriverByName('ESRI Shapefile') ds = dr.CreateDataSource(tmp_name) @@ -453,7 +464,7 @@ class ShapefileManager(ImportManager): feat = ogr.Feature(feature_def) feat.SetField('name', str(unicode_normalize(item.name)[:80])) feat.SetField('category', - str(unicode_normalize(category.name)[:80])) + str(unicode_normalize(category.name)[:80])) geom = getattr(item, geo_field.name) if not geom: @@ -479,6 +490,7 @@ class ShapefileManager(ImportManager): buff.close() return filename, zip_stream + class CSVManager(ImportManager): u""" CSV importer @@ -489,9 +501,8 @@ class CSVManager(ImportManager): # (label, getter, setter) COLS = [("Id", 'pk', 'pk'), (_(u"Name"), 'name', 'name'), - (_(u"Categories"), lambda obj:", ".join( - [c.name for c in obj.categories.all()]), - set_categories), + (_(u"Categories"), lambda obj: ", ".join( + [c.name for c in obj.categories.all()]), set_categories), (_(u"State"), 'status', lambda x: x), (_(u"Description"), 'description', 'description'), (_(u"Localisation"), 'geometry', 'geometry')] @@ -514,11 +525,11 @@ class CSVManager(ImportManager): prop_cols = [] for pm in Marker.all_properties(): prop_cols.append((pm.name, pm.getAttrName(), - pm.getAttrName()+'_set')) + pm.getAttrName() + '_set')) cols = list(self.COLS) + prop_cols - datas = [] + # datas = [] for idx, row in enumerate(reader): - if not idx: # first row + if not idx: # first row try: assert(len(row) >= len(cols)) except AssertionError: @@ -526,16 +537,17 @@ class CSVManager(ImportManager): continue if len(row) < len(cols): continue - pk, name, cats, state = row[0], row[1], row[2], row[3] + # pk, name, cats, state = row[0], row[1], row[2], row[3] + pk, name = row[0], row[1] geom = row[5] description = '' if self.importer_instance.get_description: description = row[4] COL_INDEX = 6 - dct = {'description':description, - 'name':name, - 'origin':self.importer_instance.origin, - 'license':self.importer_instance.license} + dct = {'description': description, + 'name': name, + 'origin': self.importer_instance.origin, + 'license': self.importer_instance.license} cls = None if 'POINT' in geom: cls = Marker @@ -546,8 +558,8 @@ class CSVManager(ImportManager): else: continue import_key = pk if pk else name.decode('utf-8') - item, updated, created = self.create_or_update_item(cls, dct, - import_key, pk=pk) + item, updated, created = self.create_or_update_item( + cls, dct, import_key, pk=pk) if updated: updated_item += 1 if created: @@ -555,17 +567,17 @@ class CSVManager(ImportManager): for idx, col in enumerate(cols[COL_INDEX:]): name, getter, setter_val = col setter = getattr(item, setter_val) - val = row[idx+COL_INDEX] + val = row[idx + COL_INDEX] setter(item, val) return (new_item, updated_item, msg) @classmethod def export(cls, queryset): - dct = {'description':unicode(datetime.date.today()), 'data':[]} - cls_name = queryset.model.__name__.lower() + dct = {'description': unicode(datetime.date.today()), 'data': []} + # cls_name = queryset.model.__name__.lower() cols = list(cls.COLS) for pm in queryset.model.all_properties(): - cols.append((pm.name, pm.getAttrName(), pm.getAttrName()+'_set')) + cols.append((pm.name, pm.getAttrName(), pm.getAttrName() + '_set')) header = [col[0] for col in cols] dct['data'].append(header) for item in queryset.all(): @@ -576,11 +588,12 @@ class CSVManager(ImportManager): else: data.append(getattr(item, attr)) dct['data'].append(data) - filename = unicode_normalize(settings.PROJECT_NAME + dct['description']\ + filename = unicode_normalize(settings.PROJECT_NAME + dct['description'] + '.csv') result = render_to_response('chimere/export.csv', dct) return filename, result + class GeoRSSManager(ImportManager): u""" RSS importer. @@ -596,19 +609,19 @@ class GeoRSSManager(ImportManager): - number of item updated ; - error detail on error """ - from models import Marker + from models import Marker, Route new_item, updated_item, msg = 0, 0, '' feed = feedparser.parse(self.importer_instance.source) - if feed['bozo'] and not isinstance(feed['bozo_exception'], - feedparser.CharacterEncodingOverride): + if feed['bozo'] and not isinstance( + feed['bozo_exception'], feedparser.CharacterEncodingOverride): return (0, 0, _(u"RSS feed is not well formed")) for item in feed['items']: if "georss_point" not in item and 'georss_line' not in item \ and not ("geo_lat" in item and "geo_long" in item): continue cls = None - dct = {'origin':self.importer_instance.origin, - 'license':self.importer_instance.license} + dct = {'origin': self.importer_instance.origin, + 'license': self.importer_instance.license} if 'georss_point' in item or "geo_lat" in item: cls = Marker if 'georss_point' in item: @@ -630,11 +643,11 @@ class GeoRSSManager(ImportManager): points = item['georss_line'].split(' ') reordered_points = [] # lat, lon -> x, y - for idx in xrange(len(points)/2): - reordered_points.append("%s %s" % (points[idx*2+1], - points[idx*2])) + for idx in xrange(len(points) / 2): + reordered_points.append("%s %s" % (points[idx * 2 + 1], + points[idx * 2])) dct['route'] = 'SRID=4326;LINESTRING(%s)' % \ - ",".join(reordered_points) + ",".join(reordered_points) dct['name'] = item['title'] pl_id = item['id'] if 'id' in item else item['title'] @@ -649,6 +662,7 @@ RE_HOOK = re.compile('\[([^\]]*)\]') # TODO: manage deleted item from OSM + class OSMManager(ImportManager): u""" OSM importer/exporter @@ -666,8 +680,8 @@ class OSMManager(ImportManager): - updated items; - error detail on error. """ - source, msg = self.get_source_file(['.osm'], - extra_url=self.importer_instance.filtr) + source, msg = self.get_source_file( + ['.osm'], extra_url=self.importer_instance.filtr) if not source: return (0, 0, msg) @@ -680,8 +694,8 @@ class OSMManager(ImportManager): return 0, 0, _(u"Nothing to import") def import_ways(self, tree): - from chimere.models import Marker, Route - msg, items, new_item, updated_item = "", [], 0 , 0 + from chimere.models import Route + msg, items, new_item, updated_item = "", [], 0, 0 nodes = {} for node in tree.xpath('//node'): node_id = node.attrib.get('id') @@ -703,17 +717,17 @@ class OSMManager(ImportManager): points.append(item.get('ref')) if not points: continue - wkt = 'SRID=4326;LINESTRING(%s)' % ",".join([nodes[point_id] - for point_id in points if point_id in nodes]) - dct = {'route':wkt, - 'name':name, - 'origin':self.importer_instance.origin \ - or u'OpenStreetMap.org', - 'license':self.importer_instance.license \ - or u'ODbL', - 'import_version':version} + wkt = 'SRID=4326;LINESTRING(%s)' % ",".join( + [nodes[point_id] for point_id in points if point_id in nodes]) + dct = {'route': wkt, + 'name': name, + 'origin': self.importer_instance.origin + or u'OpenStreetMap.org', + 'license': self.importer_instance.license + or u'ODbL', + 'import_version': version} item, updated, created = self.create_or_update_item( - Route, dct, node_id, version) + Route, dct, node_id, version) if updated: updated_item += 1 if created: @@ -723,7 +737,7 @@ class OSMManager(ImportManager): def import_nodes(self, tree): from chimere.models import Marker - msg, items, new_item, updated_item = "", [], 0 , 0 + msg, items, new_item, updated_item = "", [], 0, 0 for node in tree.xpath('//node'): name = None node_id = node.attrib.get('id') @@ -736,15 +750,15 @@ class OSMManager(ImportManager): name = item.attrib.get('v') point = 'SRID=4326;POINT(%s %s)' % (node.get('lon'), node.get('lat')) - dct = {'point':point, - 'name':name, - 'origin':self.importer_instance.origin \ - or u'OpenStreetMap.org', - 'license':self.importer_instance.license \ - or u'ODbL', - 'import_version':version} + dct = {'point': point, + 'name': name, + 'origin': self.importer_instance.origin + or u'OpenStreetMap.org', + 'license': self.importer_instance.license + or u'ODbL', + 'import_version': version} item, updated, created = self.create_or_update_item( - Marker, dct, node_id, version) + Marker, dct, node_id, version) if updated: updated_item += 1 if created: @@ -779,8 +793,8 @@ class OSMManager(ImportManager): username = username.encode('latin1') password = password.encode('latin1') api = OsmApi.OsmApi(api=api, username=username, password=password) - api.ChangesetCreate({u"comment": u"Import from Chimère %s" % \ - get_version()}) + api.ChangesetCreate({u"comment": u"Import from Chimère %s" % + get_version()}) hooks = RE_HOOK.findall(self.importer_instance.filtr) if not hooks: hooks = RE_HOOK.findall(self.importer_instance.source) @@ -794,28 +808,31 @@ class OSMManager(ImportManager): continue if key == 'bbox': x1, y1, x2, y2 = [float(val) for val in value.split(',')] - bbox = GEOSGeometry( + bbox = GEOSGeometry( 'POLYGON((%f %f,%f %f,%f %f,%f %f,%f %f))' % ( - x1, y1, x2, y1, x2, y2, x1, y2, x1, y1), srid=4326) + x1, y1, x2, y1, x2, y2, x1, y2, x1, y1), srid=4326) continue tags[key] = value if not tags: return 0, _(u"No non ambigious tag is defined in the XAPI request") if not bbox: - return 0, _(u"No bounding box is defined in the XAPI request."\ - u"If you are sure to manage the entire planet set the bounding box"\ - u" to -180,-90,180,90") - default_dct = {'tag':tags, - 'import_source':self.importer_instance.source} + return 0, _( + u"No bounding box is defined in the XAPI request." + u"If you are sure to manage the entire planet set the " + u"bounding box to -180,-90,180,90") + default_dct = {'tag': tags, + 'import_source': self.importer_instance.source} idx = -1 - for idx, item in enumerate(Marker.objects.filter(status='A', - point__contained=bbox, - categories=self.importer_instance.categories.all(), - not_for_osm=False, modified_since_import=True, - route=None).all()): + for idx, item in enumerate( + Marker.objects.filter( + status='A', + point__contained=bbox, + categories=self.importer_instance.categories.all(), + not_for_osm=False, modified_since_import=True, + route=None).all()): dct = default_dct.copy() - dct.update({'lon':item.point.x, - 'lat':item.point.y}) + dct.update({'lon': item.point.x, + 'lat': item.point.y}) dct['tag']['name'] = item.name node = None import_key = item.get_key('OSM') @@ -830,7 +847,7 @@ class OSMManager(ImportManager): if error.status == 404: dct.pop('id') dct.pop('version') - pass # if the node doesn't exist it is created + pass # if the node doesn't exist it is created else: raise if not updated: @@ -839,20 +856,23 @@ class OSMManager(ImportManager): item.import_version = node['version'] item.save() api.ChangesetClose() - return idx+1, None + return idx + 1, None + -import urllib2, chardet, HTMLParser +import chardet +import HTMLParser from BeautifulSoup import BeautifulSoup -from lxml import etree + RE_CLEANS = ((re.compile('(\n)*|^( )*(\n)*( )*|( )*(\n)*( )*$'), ''), (re.compile(' ( )*'), ' '), (re.compile(r"""\d{1,2}) '\ - r'(?P'+ '|'.join(UNI_MONTH_NAMES['fr_FR']) +') '\ - r'(?P\d{4})?[^\d]*'\ - r'(?P\d{1,2}) '\ - r'(?P'+ '|'.join(UNI_MONTH_NAMES['fr_FR']) +') *'\ - r'(?P\d{4})?.*'), - re.compile(r'(?P\d{1,2}) '\ - r'(?P'+ '|'.join(UNI_MONTH_NAMES['fr_FR']) +') *'\ - r'(?P\d{4})?') - ], - 'en':[ - re.compile(r'(?P\d{4})-'\ - r'(?P\d{2})-'\ - r'(?P\d{2})'\ - r'(?:T'\ - r'(?P\d{2})?:'\ - r'(?P\d{2})?:'\ - r'(?P\d{2})'\ - r')?.*'\ - r'(?P\d{4})-'\ - r'(?P\d{2})-'\ - r'(?P\d{2})'\ - r'(?:T'\ - r'(?P\d{2})?:'\ - r'(?P\d{2})?:'\ - r'(?P\d{2})'\ - r')?.*' - ), - re.compile(r'(?P\d{4})-'\ - r'(?P\d{2})-'\ - r'(?P\d{2})'\ - r'(?:T'\ - r'(?P\d{2})?:'\ - r'(?P\d{2})?:'\ - r'(?P\d{2})'\ - r')?' - ) - ], - } + UNI_MONTH_NAMES = {locale: [m for m in MONTH_NAMES[locale]] + for locale in MONTH_NAMES} + +DATE_PARSINGS = { + 'fr_FR': [ + re.compile(r'(?P\d{1,2}) ' + r'(?P' + '|'.join(UNI_MONTH_NAMES['fr_FR']) + ') ' + r'(?P\d{4})?[^\d]*' + r'(?P\d{1,2}) ' + r'(?P' + '|'.join(UNI_MONTH_NAMES['fr_FR']) + ') *' + r'(?P\d{4})?.*'), + re.compile(r'(?P\d{1,2}) ' + r'(?P' + '|'.join(UNI_MONTH_NAMES['fr_FR']) + ') * ' + r'(?P\d{4})?')], + 'en': [ + re.compile(r'(?P\d{4})-' + r'(?P\d{2})-' + r'(?P\d{2})' + r'(?:T' + r'(?P\d{2})?:' + r'(?P\d{2})?:' + r'(?P\d{2})' + r')?.*' + r'(?P\d{4})-' + r'(?P\d{2})-' + r'(?P\d{2})' + r'(?:T' + r'(?P\d{2})?:' + r'(?P\d{2})?:' + r'(?P\d{2})' + r')?.*'), + re.compile(r'(?P\d{4})-' + r'(?P\d{2})-' + r'(?P\d{2})' + r'(?:T' + r'(?P\d{2})?:' + r'(?P\d{2})?:' + r'(?P\d{2})' + r')?')], +} + def clean_field(value): return value.strip() + class HtmlXsltManager(ImportManager): PARSER = 'HTMLParser' + def get(self): u""" Get data from the source @@ -939,7 +959,7 @@ class HtmlXsltManager(ImportManager): soup = BeautifulSoup(data) main_page = soup.prettify() # convert it to valid XHTML - #doc, errors = tidy_document(main_page) + # doc, errors = tidy_document(main_page) doc = main_page dom = etree.HTML(doc, getattr(etree, self.PARSER)()) try: @@ -963,8 +983,8 @@ class HtmlXsltManager(ImportManager): base_url = u"/".join(self.importer_instance.source.split(u'/')[:-1]) base_url += u"/" for item in newdom.getroot(): - c_item = {child.tag:clean_field(child.text) - for child in item.getchildren() if child.text} + c_item = {child.tag: clean_field(child.text) + for child in item.getchildren() if child.text} # try to have more information on the linked page if transform_child and 'link' in c_item: # not an absolute address @@ -985,8 +1005,8 @@ class HtmlXsltManager(ImportManager): child_dom = etree.HTML(child_page, etree.HTMLParser()) extra_keys = transform_child(child_dom).getroot() if len(extra_keys): - c_item.update({extra.tag:etree.tostring(extra) - for extra in extra_keys[0].getchildren()}) + c_item.update({extra.tag: etree.tostring(extra) + for extra in extra_keys[0].getchildren()}) items.append(c_item) # change relative link to full link, simplify, unescape HTML entities html_unescape = HTMLParser.HTMLParser().unescape @@ -994,7 +1014,7 @@ class HtmlXsltManager(ImportManager): for k in item: val = item[k] for r, replaced in RE_CLEANS: - val = re.sub(r, replaced % {'base_url':base_url}, val) + val = re.sub(r, replaced % {'base_url': base_url}, val) item[k] = html_unescape(val) self.key_categories = self.importer_instance.get_key_category_dict() self.missing_cats = set() @@ -1003,9 +1023,10 @@ class HtmlXsltManager(ImportManager): self.add_dct_item(item) msg = '' if self.missing_cats: - msg = _(u"Names \"%s\" doesn't match existing categories. " - u"Modify the import to match theses names with categories.") % ( - u'", "'.join(self.missing_cats)) + msg = _( + u"Names \"%s\" doesn't match existing categories. " + u"Modify the import to match theses names with categories.") %\ + (u'", "'.join(self.missing_cats)) return (self.new_item, self.updated_item, msg) @classmethod @@ -1042,18 +1063,18 @@ class HtmlXsltManager(ImportManager): if not m: continue values = m.groupdict() - date = self._internal_parse_date(locale, - 'year1' in values and values['year1'], - values['month1'], values['day1']) + date = self._internal_parse_date( + locale, 'year1' in values and values['year1'], + values['month1'], values['day1']) if not date: continue dct['start_date'] = date has_dates = True if 'day2' not in values: break - date = self._internal_parse_date(locale, - 'year2' in values and values['year2'], - values['month2'], values['day2']) + date = self._internal_parse_date( + locale, 'year2' in values and values['year2'], + values['month2'], values['day2']) if date: dct['end_date'] = date break @@ -1061,14 +1082,14 @@ class HtmlXsltManager(ImportManager): def add_dct_item(self, item): if not self.importer_instance.default_localisation and \ - not "point" in item and not ("lat" in item and item['lat']): + "point" not in item and not ("lat" in item and item['lat']): return cls = None - dct = {'origin':"%s" % ( - item.get('link') or '#', - self.importer_instance.origin), - 'license':self.importer_instance.license, - 'name':item['name']} + dct = { + 'origin': "%s" % ( + item.get('link') or '#', self.importer_instance.origin), + 'license': self.importer_instance.license, + 'name': item['name']} category = None if 'category' in item and item['category']: if item['category'] in self.key_categories: @@ -1095,5 +1116,6 @@ class HtmlXsltManager(ImportManager): if created: self.new_item += 1 + class XMLXsltManager(HtmlXsltManager): PARSER = 'XMLParser' -- cgit v1.2.3 From ea17ebad02a419da21c4512954c32dfb5ca7b650 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Sun, 22 Nov 2015 13:16:54 +0100 Subject: Manage JSON imports --- .../0012_auto__chg_field_importer_filtr.py | 290 +++++++++++++++++++++ chimere/models.py | 24 +- chimere/static/chimere/js/importer_interface.js | 7 +- chimere/tests.py | 16 ++ chimere/tests/test.json | 57 ++++ chimere/utils.py | 72 +++++ 6 files changed, 455 insertions(+), 11 deletions(-) create mode 100644 chimere/migrations/0012_auto__chg_field_importer_filtr.py create mode 100644 chimere/tests/test.json (limited to 'chimere/utils.py') diff --git a/chimere/migrations/0012_auto__chg_field_importer_filtr.py b/chimere/migrations/0012_auto__chg_field_importer_filtr.py new file mode 100644 index 0000000..84e4c72 --- /dev/null +++ b/chimere/migrations/0012_auto__chg_field_importer_filtr.py @@ -0,0 +1,290 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Changing field 'Importer.filtr' + db.alter_column('chimere_importer', 'filtr', self.gf('django.db.models.fields.TextField')(null=True)) + + def backwards(self, orm): + + # Changing field 'Importer.filtr' + db.alter_column('chimere_importer', 'filtr', self.gf('django.db.models.fields.CharField')(max_length=200, null=True)) + + models = { + 'chimere.aggregatedroute': { + 'Meta': {'object_name': 'AggregatedRoute', 'db_table': "'chimere_aggregated_routes'", 'managed': 'False'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'route': ('django.contrib.gis.db.models.fields.MultiLineStringField', [], {}), + 'status': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'subcategory': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.SubCategory']"}) + }, + 'chimere.area': { + 'Meta': {'ordering': "('order', 'name')", 'object_name': 'Area'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'default': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'default_subcategories': ('chimere.widgets.SelectMultipleField', [], {'to': "orm['chimere.SubCategory']", 'symmetrical': 'False', 'blank': 'True'}), + 'dynamic_categories': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'external_css': ('django.db.models.fields.URLField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'layers': ('chimere.widgets.SelectMultipleField', [], {'symmetrical': 'False', 'related_name': "'areas'", 'blank': 'True', 'through': "orm['chimere.AreaLayers']", 'to': "orm['chimere.Layer']"}), + 'lower_right_corner': ('django.contrib.gis.db.models.fields.PointField', [], {'default': "'POINT(0 0)'"}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {'unique': 'True'}), + 'restrict_to_extent': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'subcategories': ('chimere.widgets.SelectMultipleField', [], {'symmetrical': 'False', 'related_name': "'areas'", 'blank': 'True', 'db_table': "'chimere_subcategory_areas'", 'to': "orm['chimere.SubCategory']"}), + 'upper_left_corner': ('django.contrib.gis.db.models.fields.PointField', [], {'default': "'POINT(0 0)'"}), + 'urn': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '50', 'blank': 'True'}), + 'welcome_message': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, + 'chimere.arealayers': { + 'Meta': {'ordering': "('order',)", 'object_name': 'AreaLayers'}, + 'area': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Area']"}), + 'default': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'layer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Layer']"}), + 'order': ('django.db.models.fields.IntegerField', [], {}) + }, + 'chimere.category': { + 'Meta': {'ordering': "['order']", 'object_name': 'Category'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {}) + }, + 'chimere.color': { + 'Meta': {'ordering': "['order']", 'object_name': 'Color'}, + 'code': ('django.db.models.fields.CharField', [], {'max_length': '6'}), + 'color_theme': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.ColorTheme']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'order': ('django.db.models.fields.IntegerField', [], {}) + }, + 'chimere.colortheme': { + 'Meta': {'object_name': 'ColorTheme'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}) + }, + 'chimere.icon': { + 'Meta': {'object_name': 'Icon'}, + 'height': ('django.db.models.fields.IntegerField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'image': ('django.db.models.fields.files.ImageField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'width': ('django.db.models.fields.IntegerField', [], {}) + }, + 'chimere.importer': { + 'Meta': {'object_name': 'Importer'}, + 'associate_marker_to_way': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'automatic_update': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'categories': ('chimere.widgets.SelectMultipleField', [], {'symmetrical': 'False', 'to': "orm['chimere.SubCategory']", 'null': 'True', 'blank': 'True'}), + 'default_description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'default_localisation': ('chimere.widgets.PointField', [], {'null': 'True', 'blank': 'True'}), + 'default_name': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'default_status': ('django.db.models.fields.CharField', [], {'default': "'I'", 'max_length': '1'}), + 'filtr': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'get_description': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'importer_type': ('django.db.models.fields.CharField', [], {'max_length': '4'}), + 'license': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'origin': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'overwrite': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'source': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'source_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'source_file_alt': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'srid': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'state': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'zipped': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + }, + 'chimere.importerkeycategories': { + 'Meta': {'object_name': 'ImporterKeyCategories'}, + 'category': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.SubCategory']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'importer': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'key_categories'", 'to': "orm['chimere.Importer']"}), + 'key': ('django.db.models.fields.CharField', [], {'max_length': '200'}) + }, + 'chimere.layer': { + 'Meta': {'object_name': 'Layer'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'layer_code': ('django.db.models.fields.TextField', [], {'max_length': '300'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}) + }, + 'chimere.marker': { + 'Meta': {'ordering': "('status', 'name')", 'object_name': 'Marker'}, + 'available_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'categories': ('chimere.widgets.SelectMultipleField', [], {'to': "orm['chimere.SubCategory']", 'symmetrical': 'False'}), + 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'import_key': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'import_source': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'import_version': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'is_front_page': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'keywords': ('django.db.models.fields.TextField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'license': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'modified_since_import': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'not_for_osm': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'origin': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'point': ('chimere.widgets.PointField', [], {}), + 'ref_item': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'submited_marker'", 'null': 'True', 'to': "orm['chimere.Marker']"}), + 'route': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'associated_marker'", 'null': 'True', 'to': "orm['chimere.Route']"}), + 'start_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'submiter_comment': ('django.db.models.fields.TextField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'submiter_email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'submiter_name': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}), + 'submiter_session_key': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}) + }, + 'chimere.multimediaextension': { + 'Meta': {'object_name': 'MultimediaExtension'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'multimedia_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'extensions'", 'to': "orm['chimere.MultimediaType']"}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '6'}) + }, + 'chimere.multimediafile': { + 'Meta': {'object_name': 'MultimediaFile'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'marker': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'multimedia_files'", 'to': "orm['chimere.Marker']"}), + 'miniature': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'multimedia_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.MultimediaType']", 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {'default': '1'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '200'}) + }, + 'chimere.multimediatype': { + 'Meta': {'object_name': 'MultimediaType'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'iframe': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'media_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'mime_type': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}) + }, + 'chimere.news': { + 'Meta': {'object_name': 'News'}, + 'areas': ('chimere.widgets.SelectMultipleField', [], {'symmetrical': 'False', 'to': "orm['chimere.Area']", 'null': 'True', 'blank': 'True'}), + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'content': ('django.db.models.fields.TextField', [], {}), + 'date': ('django.db.models.fields.DateField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_front_page': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}) + }, + 'chimere.page': { + 'Meta': {'object_name': 'Page'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'content': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mnemonic': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}), + 'order': ('django.db.models.fields.IntegerField', [], {'default': '10', 'null': 'True', 'blank': 'True'}), + 'template_path': ('django.db.models.fields.CharField', [], {'max_length': '150', 'null': 'True', 'blank': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '150'}) + }, + 'chimere.picturefile': { + 'Meta': {'object_name': 'PictureFile'}, + 'height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'marker': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'pictures'", 'to': "orm['chimere.Marker']"}), + 'miniature': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {'default': '1'}), + 'picture': ('django.db.models.fields.files.ImageField', [], {'max_length': '100'}), + 'thumbnailfile': ('django.db.models.fields.files.ImageField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'thumbnailfile_height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'thumbnailfile_width': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'width': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}) + }, + 'chimere.property': { + 'Meta': {'object_name': 'Property'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'marker': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Marker']"}), + 'propertymodel': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.PropertyModel']"}), + 'value': ('django.db.models.fields.TextField', [], {}) + }, + 'chimere.propertymodel': { + 'Meta': {'ordering': "('order',)", 'object_name': 'PropertyModel'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mandatory': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {}), + 'subcategories': ('chimere.widgets.SelectMultipleField', [], {'symmetrical': 'False', 'related_name': "'properties'", 'blank': 'True', 'to': "orm['chimere.SubCategory']"}), + 'type': ('django.db.models.fields.CharField', [], {'max_length': '1'}) + }, + 'chimere.propertymodelchoice': { + 'Meta': {'object_name': 'PropertyModelChoice'}, + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'propertymodel': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'choices'", 'to': "orm['chimere.PropertyModel']"}), + 'value': ('django.db.models.fields.CharField', [], {'max_length': '150'}) + }, + 'chimere.route': { + 'Meta': {'ordering': "('status', 'name')", 'object_name': 'Route'}, + 'associated_file': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.RouteFile']", 'null': 'True', 'blank': 'True'}), + 'categories': ('chimere.widgets.SelectMultipleField', [], {'to': "orm['chimere.SubCategory']", 'symmetrical': 'False'}), + 'end_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'has_associated_marker': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'height': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'import_key': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'import_source': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'import_version': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'keywords': ('django.db.models.fields.TextField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'license': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'modified_since_import': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'not_for_osm': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'origin': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'picture': ('django.db.models.fields.files.ImageField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'ref_item': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'submited_route'", 'null': 'True', 'to': "orm['chimere.Route']"}), + 'route': ('chimere.widgets.RouteField', [], {}), + 'start_date': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'submiter_comment': ('django.db.models.fields.TextField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'submiter_email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'submiter_name': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}), + 'submiter_session_key': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}), + 'width': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}) + }, + 'chimere.routefile': { + 'Meta': {'ordering': "('name',)", 'object_name': 'RouteFile'}, + 'file_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'raw_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}), + 'simplified_file': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}) + }, + 'chimere.subcategory': { + 'Meta': {'ordering': "['category', 'order']", 'object_name': 'SubCategory'}, + 'as_layer': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'category': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'subcategories'", 'to': "orm['chimere.Category']"}), + 'color_theme': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.ColorTheme']", 'null': 'True', 'blank': 'True'}), + 'dated': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'hover_icon': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'subcat_hovered'", 'null': 'True', 'to': "orm['chimere.Icon']"}), + 'icon': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['chimere.Icon']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'item_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}), + 'keywords': ('django.db.models.fields.TextField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '150'}), + 'order': ('django.db.models.fields.IntegerField', [], {'default': '1000'}), + 'routing_warn': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'submission': ('django.db.models.fields.BooleanField', [], {'default': 'True'}) + }, + 'chimere.tinyurl': { + 'Meta': {'object_name': 'TinyUrl'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'parameters': ('django.db.models.fields.CharField', [], {'max_length': '500'}) + } + } + + complete_apps = ['chimere'] \ No newline at end of file diff --git a/chimere/models.py b/chimere/models.py index be84036..e7a0f8f 100644 --- a/chimere/models.py +++ b/chimere/models.py @@ -46,7 +46,8 @@ from chimere.widgets import HiddenPointChooserWidget, PointField, RouteField, \ DatePickerWidget from chimere.managers import BaseGeoManager from chimere.utils import KMLManager, OSMManager, ShapefileManager, \ - GeoRSSManager, CSVManager, HtmlXsltManager, XMLXsltManager + GeoRSSManager, CSVManager, HtmlXsltManager, XMLXsltManager, JsonManager + class Page(models.Model): """Simple extra pages @@ -316,13 +317,14 @@ STATUS = (('S', _(u'Submited')), ('I', _(u'Imported'))) STATUS_DCT = dict(STATUS) -IMPORTERS = {'KML':KMLManager, - 'OSM':OSMManager, - 'SHP':ShapefileManager, - 'RSS':GeoRSSManager, - 'CSV':CSVManager, - 'XSLT':HtmlXsltManager, - 'XXLT':XMLXsltManager +IMPORTERS = {'KML': KMLManager, + 'OSM': OSMManager, + 'SHP': ShapefileManager, + 'RSS': GeoRSSManager, + 'CSV': CSVManager, + 'JSON': JsonManager, + 'XSLT': HtmlXsltManager, + 'XXLT': XMLXsltManager } IMPORTER_CHOICES = (('KML', 'KML'), @@ -330,20 +332,21 @@ IMPORTER_CHOICES = (('KML', 'KML'), ('SHP', 'Shapefile'), ('RSS', 'GeoRSS'), ('CSV', 'CSV'), + ('JSON', 'JSON'), ('XSLT', 'HTML-XSLT'), ('XXLT', 'XML-XSLT'), ) IMPORTER_CHOICES_DICT = dict(IMPORTER_CHOICES) + class Importer(models.Model): ''' Data importer for a specific subcategory ''' importer_type = models.CharField(_(u"Importer type"), max_length=4, choices=IMPORTER_CHOICES) - filtr = models.CharField(_(u"Filter"), max_length=200, - blank=True, null=True) + filtr = models.TextField(_(u"Filter"), blank=True, null=True) source = models.CharField(_(u"Web address"), max_length=200, blank=True, null=True, help_text=_(u"Don't forget the trailing slash")) @@ -410,6 +413,7 @@ class Importer(models.Model): dct[key_cat.key] = key_cat.category return dct + class ImporterKeyCategories(models.Model): """ Association between key and categories diff --git a/chimere/static/chimere/js/importer_interface.js b/chimere/static/chimere/js/importer_interface.js index 5c77a8d..c2a3a07 100644 --- a/chimere/static/chimere/js/importer_interface.js +++ b/chimere/static/chimere/js/importer_interface.js @@ -32,7 +32,12 @@ django.jQuery(function($) { 'field-origin', 'field-srid', 'field-license', 'field-categories', 'field-overwrite', 'field-get_description', 'field-default_localisation', - 'field-automatic_update', 'field-default_status') + 'field-automatic_update', 'field-default_status'), + JSON:new Array('field-source', 'field-source_file', 'field-default_name', + 'field-filtr', 'field-zipped', 'field-origin', + 'field-license', 'field-categories', 'field-overwrite', + 'field-get_description', 'field-automatic_update', + 'field-default_status', 'field-default_localisation') } var osm_map_initialized; var edit_map_initialized; diff --git a/chimere/tests.py b/chimere/tests.py index 3a3144e..6c39cba 100644 --- a/chimere/tests.py +++ b/chimere/tests.py @@ -323,6 +323,22 @@ class XmlXsltImporterTest(TestCase, ImporterTest): importer1.categories.add(subcategories[0]) self.marker_importers = [(importer1, 10),] + +class JsonImporterTest(TestCase, ImporterTest): + def setUp(self): + subcategories = subcategory_setup() + jsonfile = File(open(test_dir_path + 'tests/test.json')) + importer1 = Importer.objects.create( + importer_type='JSON', + source_file=jsonfile, + filtr='{"title":"name", "id_agenda":"id", ' + '"content":"description", "date_start_evt":"start_date", ' + '"date_end_evt":"end_date"}', + default_localisation='SRID=4326;POINT(-4.5 48.4)',) + importer1.categories.add(subcategories[0]) + self.marker_importers = [(importer1, 5), ] + + class FeedsTest(TestCase): def setUp(self): self.areas = areas_setup() diff --git a/chimere/tests/test.json b/chimere/tests/test.json new file mode 100644 index 0000000..6c9fa0c --- /dev/null +++ b/chimere/tests/test.json @@ -0,0 +1,57 @@ + +[ +{"id_agenda":"1759", +"title":"Exposition municipale d'Orsay : inscription en ligne", +"content":"TEST +TESTST", +"date_start_evt":"2015-08-20", +"date_end_evt":"2015-12-14", +"date_start_display":"2015-10-20", +"date_end_display":"2015-12-14", +"date_add":"2015-10-20 10:37:52", +"date_update":"2015-10-27 14:56:18" +}, +{"id_agenda":"179", +"title":"Exposition municipale d'Orsay : inscription en ligne", +"content":"TEST +TESTST", +"date_start_evt":"2015-08-20", +"date_end_evt":"2015-12-14", +"date_start_display":"2015-10-20", +"date_end_display":"2015-12-14", +"date_add":"2015-10-20 10:37:52", +"date_update":"2015-10-27 14:56:18" +}, +{"id_agenda":"19", +"title":"Exposition municipale d'Orsay : inscription en ligne", +"content":"TEST +TESTST", +"date_start_evt":"2015-08-20", +"date_end_evt":"2015-12-14", +"date_start_display":"2015-10-20", +"date_end_display":"2015-12-14", +"date_add":"2015-10-20 10:37:52", +"date_update":"2015-10-27 14:56:18" +}, +{"id_agenda":"17", +"title":"Exposition municipale d'Orsay : inscription en ligne", +"content":"TEST +TESTST", +"date_start_evt":"2015-08-20", +"date_end_evt":"2015-12-14", +"date_start_display":"2015-10-20", +"date_end_display":"2015-12-14", +"date_add":"2015-10-20 10:37:52", +"date_update":"2015-10-27 14:56:18" +}, +{"id_agenda":"59", +"title":"Exposition municipale d'Orsay : inscription en ligne", +"content":"TEST +TESTST", +"date_start_evt":"2015-08-20", +"date_end_evt":"2015-12-14", +"date_start_display":"2015-10-20", +"date_end_display":"2015-12-14", +"date_add":"2015-10-20 10:37:52", +"date_update":"2015-10-27 14:56:18" +}] diff --git a/chimere/utils.py b/chimere/utils.py index e017762..5289600 100644 --- a/chimere/utils.py +++ b/chimere/utils.py @@ -27,6 +27,7 @@ import feedparser import os import re import StringIO +import json import tempfile import urllib2 import unicodedata @@ -658,6 +659,77 @@ class GeoRSSManager(ImportManager): new_item += 1 return (new_item, updated_item, msg) + +class JsonManager(ImportManager): + u""" + Json importer. + This manager only gets and do not produce Json feed + """ + + def get(self): + u""" + Get data from a json simple source + + Return a tuple with: + - number of new item ; + - number of item updated ; + - error detail on error + """ + from models import Marker + new_item, updated_item, msg = 0, 0, '' + source, msg = self.get_source_file(['.json']) + if msg: + return (0, 0, msg) + + vals = source.read().replace('\n', ' ') + try: + values = json.JSONDecoder().decode(vals) + except ValueError as e: + return (new_item, updated_item, + _(u"JSON file is not well formed: " + e.message)) + # configuration in filtr + try: + filtr = json.JSONDecoder().decode(self.importer_instance.filtr) + except ValueError: + return ( + new_item, updated_item, + _(u"Bad configuration: filter field must be a valid " + u"JSON string")) + + vls = filtr.values() + for k in ('name', 'id', 'description'): + if k not in vls: + return ( + new_item, updated_item, + _(u"A key must be associated to \"%s\" in the " + u"filter.") % k) + + for item in values: + dct = {'origin': self.importer_instance.origin, + 'license': self.importer_instance.license} + for k in filtr: + if k in item and item[k]: + dct[filtr[k]] = item[k] + if 'point' in item: + x, y = item['point'].split(",") + dct['point'] = 'SRID=4326;POINT(%s %s)' % (x, y) + elif 'lat' in item and item['lat']: + dct['point'] = 'SRID=4326;POINT(%s %s)' % (item['lon'], + item['lat']) + else: + dct['point'] = self.importer_instance.default_localisation + if not dct['point']: + continue + cls = Marker + pl_id = (dct.pop('id') if 'id' in dct else dct['name']) \ + + "-" + unicode(self.importer_instance.pk) + it, updated, created = self.create_or_update_item(cls, dct, pl_id) + if updated: + updated_item += 1 + if created: + new_item += 1 + return (new_item, updated_item, msg) + RE_HOOK = re.compile('\[([^\]]*)\]') # TODO: manage deleted item from OSM -- cgit v1.2.3 From a6be0e9966c51aef1ef1b154d28627924902e179 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Mon, 28 Dec 2015 09:30:16 +0100 Subject: Better management of JSON imports --- chimere/utils.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) (limited to 'chimere/utils.py') diff --git a/chimere/utils.py b/chimere/utils.py index 5289600..0dfe588 100644 --- a/chimere/utils.py +++ b/chimere/utils.py @@ -22,12 +22,13 @@ Utilitaries """ import csv +import collections import datetime import feedparser +import json import os import re import StringIO -import json import tempfile import urllib2 import unicodedata @@ -683,7 +684,8 @@ class JsonManager(ImportManager): vals = source.read().replace('\n', ' ') try: - values = json.JSONDecoder().decode(vals) + values = json.JSONDecoder( + object_pairs_hook=collections.OrderedDict).decode(vals) except ValueError as e: return (new_item, updated_item, _(u"JSON file is not well formed: " + e.message)) @@ -704,20 +706,38 @@ class JsonManager(ImportManager): _(u"A key must be associated to \"%s\" in the " u"filter.") % k) + default_dct = {'origin': self.importer_instance.origin, + 'license': self.importer_instance.license} + if 'prefix_title' in filtr: + default_dct['title'] = filtr.pop('prefix_title') + if 'prefix_description' in filtr: + default_dct['description'] = filtr.pop('prefix_description') + if self.importer_instance.default_localisation: + default_dct['point'] = self.importer_instance.default_localisation + for item in values: - dct = {'origin': self.importer_instance.origin, - 'license': self.importer_instance.license} + dct = default_dct.copy() for k in filtr: if k in item and item[k]: - dct[filtr[k]] = item[k] + if filtr[k] not in dct: + dct[filtr[k]] = "" + else: + if filtr[k] == 'description': + dct[filtr[k]] += "
" + else: + dct[filtr[k]] += " " + dct[filtr[k]] += item[k] if 'point' in item: x, y = item['point'].split(",") dct['point'] = 'SRID=4326;POINT(%s %s)' % (x, y) - elif 'lat' in item and item['lat']: + elif 'lat' in item and item['lat'] \ + and 'lon' in item and item['lon']: dct['point'] = 'SRID=4326;POINT(%s %s)' % (item['lon'], item['lat']) - else: - dct['point'] = self.importer_instance.default_localisation + elif 'x' in item and item['x'] \ + and 'y' in item and item['y']: + dct['point'] = 'SRID=4326;POINT(%s %s)' % (item['x'], + item['y']) if not dct['point']: continue cls = Marker -- cgit v1.2.3 From 8acc204d20d97b6cf909d51afd0d4089bb86e132 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Mon, 28 Dec 2015 09:39:20 +0100 Subject: Import JSON: prefix_title -> prefix_name --- chimere/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'chimere/utils.py') diff --git a/chimere/utils.py b/chimere/utils.py index 0dfe588..4944d73 100644 --- a/chimere/utils.py +++ b/chimere/utils.py @@ -708,8 +708,8 @@ class JsonManager(ImportManager): default_dct = {'origin': self.importer_instance.origin, 'license': self.importer_instance.license} - if 'prefix_title' in filtr: - default_dct['title'] = filtr.pop('prefix_title') + if 'prefix_name' in filtr: + default_dct['name'] = filtr.pop('prefix_name') if 'prefix_description' in filtr: default_dct['description'] = filtr.pop('prefix_description') if self.importer_instance.default_localisation: -- cgit v1.2.3 From 89c4baf537a590dbe80c5f6a4c20f202c63ebb15 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Mon, 4 Jan 2016 02:07:07 +0100 Subject: Fix XHTML importer when no description is provided --- chimere/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'chimere/utils.py') diff --git a/chimere/utils.py b/chimere/utils.py index 4944d73..8066255 100644 --- a/chimere/utils.py +++ b/chimere/utils.py @@ -1197,7 +1197,7 @@ class HtmlXsltManager(ImportManager): item['lat']) else: dct['point'] = self.importer_instance.default_localisation - dct['description'] = item['description'] + dct['description'] = item.get('description', '') if 'date' in item: dct.update(self.parse_date(item['date'])) key = item['key'] -- cgit v1.2.3