diff options
Diffstat (limited to 'chimere/utils.py')
| -rw-r--r-- | chimere/utils.py | 272 |
1 files changed, 202 insertions, 70 deletions
diff --git a/chimere/utils.py b/chimere/utils.py index 4cd6f45..75c0ad4 100644 --- a/chimere/utils.py +++ b/chimere/utils.py @@ -23,6 +23,7 @@ Utilitaries import csv import datetime +import feedparser import os import re import StringIO @@ -69,16 +70,21 @@ class ImportManager: def put(self): pass - def create_or_update_item(self, cls, values, import_key, version=None): + def create_or_update_item(self, cls, values, import_key, version=None, + key='', pk=None): updated, created, item = False, False, None - if import_key: + import_key = unicode(import_key).replace(':', '^') + if not key: + key = self.importer_instance.importer_type + if import_key or pk: dct_import = { - 'import_key__icontains':'%s:%s;' % ( - self.importer_instance.importer_type, - import_key), + 'import_key__icontains':'%s:%s;' % (key, import_key), 'import_source':self.importer_instance.source} try: - item = cls.objects.get(**dct_import) + if pk: + item = cls.objects.get(pk=pk) + else: + item = cls.objects.get(**dct_import) if version and item.import_version == int(version): # no update since the last import return item, None, None @@ -103,8 +109,7 @@ class ImportManager: return None, False, False created = True if import_key: - item.set_key(self.importer_instance.importer_type, - import_key) + item.set_key(key, import_key) item.categories.clear() for cat in self.importer_instance.categories.all(): item.categories.add(cat) @@ -138,8 +143,9 @@ class ImportManager: files.append(None) return files - def get_source_file(self, source, suffixes, dest_dir=None, + def get_source_file(self, suffixes, dest_dir=None, extra_url=None): + source = self.importer_instance.source_file if not hasattr(source, 'read'): if not source: source = self.importer_instance.source \ @@ -157,7 +163,7 @@ class ImportManager: source = open(source) except IOError, msg: return (None, msg) - except urllib2.URLError as error: + except (urllib2.URLError, AttributeError) as error: return (None, error.message) if self.importer_instance.zipped: try: @@ -182,21 +188,18 @@ class KMLManager(ImportManager): self.importer_instance = importer_instance self.ns = ns - def get(self, source=None): + def get(self): u""" - Get data from the source - Args: - - source (None): input file if not provided get it from the distant - source provided in the importer instance. + Get data from a KML source Return a tuple with: - number of new item ; - number of item updated ; - error detail on error """ - from models import Marker + from models import Marker, Route new_item, updated_item, msg = 0, 0, '' - source, msg = self.get_source_file(source, ['.kml']) + source, msg = self.get_source_file(['.kml']) if msg: return (0, 0, msg) doc = source @@ -207,7 +210,10 @@ class KMLManager(ImportManager): if line.strip(): break doc = StringIO.StringIO("\n".join(splitted[idx:])) - tree = etree.parse(doc) + try: + tree = etree.parse(doc) + except: + return (0, 0, _(u"Bad XML file")) # try to get default namespace if not self.ns: self.ns = tree.getroot().nsmap[None] @@ -215,7 +221,7 @@ class KMLManager(ImportManager): if self.importer_instance.filtr else self.DEFAULT_XPATH for placemark in tree.xpath(xpath, namespaces={'kml':self.ns}): - name, point = None, None + name, point, line = None, None, None pl_id = placemark.attrib.get('id') pl_key = 'kml-%d' % self.importer_instance.pk ns = '{%s}' % self.ns @@ -232,34 +238,32 @@ class KMLManager(ImportManager): if coord.tag == ns + 'coordinates': x, y, z = coord.text.split(',') point = 'SRID=4326;POINT(%s %s)' % (x, y) + elif item.tag == ns + 'LineString': + for coord in item: + if coord.tag == ns + 'coordinates': + points = coord.text.replace('\n', ' ').split(' ') + points = ",".join([" ".join(p.split(',')[:2]) + for p in points if p]) + line = 'SRID=4326;LINESTRING(%s)' % points + cls = None + dct = {'description':description, + 'name':name, + 'origin':self.importer_instance.origin, + 'license':self.importer_instance.license} if point: - dct = {'point':point, - 'description':description, - 'name':name,} - m = None - if pl_id: - dct_import = { - 'import_key__icontains':'%s:%s;' % (pl_key, pl_id), - 'import_source':self.importer_instance.source} - try: - m = Marker.objects.get(**dct_import) - for k in dct: - setattr(m, k, dct[k]) - m.save() - updated_item += 1 - except ObjectDoesNotExist: - m = None - dct.update({ - 'import_source':self.importer_instance.source}) - if not m: - dct['status'] = 'I' - m = Marker.objects.create(**dct) + dct['point'] = point + cls = Marker + if line: + dct['route'] = line + dct.pop('description') + cls = Route + if cls: + item, updated, created = self.create_or_update_item( + cls, dct, pl_id, key=pl_key) + if updated: + updated_item += 1 + if created: new_item += 1 - if pl_id: - m.set_key(pl_key, pl_id) - m.categories.clear() - for cat in self.importer_instance.categories.all(): - m.categories.add(cat) return (new_item, updated_item, msg) @classmethod @@ -277,12 +281,9 @@ class ShapefileManager(ImportManager): u""" Shapefile importer """ - def get(self, source=None): + def get(self): u""" - Get data from the source - Args: - - source (None): input file if not provided get it from the distant - source provided in the importer instance. + Get data from a Shapefile source Return a tuple with: - number of new item ; @@ -292,8 +293,7 @@ class ShapefileManager(ImportManager): from models import Marker, Route new_item, updated_item, msg = 0, 0, '' tmpdir = tempfile.mkdtemp() - sources, msg = self.get_source_file(source, - ['.shp', '.dbf', '.prj', '.shx'], + sources, msg = self.get_source_file(['.shp', '.dbf', '.prj', '.shx'], dest_dir=tmpdir) if msg: return (0, 0, msg) @@ -350,13 +350,18 @@ class ShapefileManager(ImportManager): name.decode(settings.CHIMERE_SHAPEFILE_ENCODING)) except: continue - geoms = [feat.geom.wkt] + try: + geoms = [feat.geom.wkt] + except: + return (0, 0, _(u"Bad Shapefile")) if feat.geom.geom_type == 'MultiLineString': geoms = [geom.wkt for geom in feat.geom] import_key = feat.get(id_name) if id_name and len(geoms) == 1 else '' for geom in geoms: dct = {geom_key:'SRID=%s;%s' % (srid, geom), - 'name':name + 'name':name, + 'origin':self.importer_instance.origin, + 'license':self.importer_instance.license } item, updated, created = self.create_or_update_item( geom_cls, dct, import_key) @@ -441,26 +446,91 @@ class CSVManager(ImportManager): u""" CSV importer """ - _COLS = [("Id", 'pk'), (_(u"Name"), 'name'), + @classmethod + def set_categories(value): + return + + # (label, getter, setter) + COLS = [("Id", 'pk', 'pk'), (_(u"Name"), 'name', 'name'), (_(u"Categories"), lambda obj:", ".join( - [c.name for c in obj.categories.all()])), - (_(u"State"), 'status')] - COLS = {'marker':_COLS+[(_(u"Description"), 'description'), - (_(u"Localisation"), lambda obj: obj.point.wkt)], - 'route':_COLS+[(_(u"Localisation"), lambda obj: obj.route.wkt)]} + [c.name for c in obj.categories.all()]), + set_categories), + (_(u"State"), 'status', lambda x: x), + (_(u"Description"), 'description', 'description'), + (_(u"Localisation"), 'geometry', 'geometry')] + + def get(self): + u""" + Get data from a CSV source + + Return a tuple with: + - number of new item ; + - number of item updated ; + - error detail on error + """ + from models import Marker, Route + new_item, updated_item, msg = 0, 0, '' + source, msg = self.get_source_file(['.csv']) + if msg: + return (0, 0, msg) + reader = csv.reader(source, delimiter=';', quotechar='"') + prop_cols = [] + for pm in Marker.properties(): + prop_cols.append((pm.name, pm.getAttrName(), + pm.getAttrName()+'_set')) + cols = self.COLS + prop_cols + datas = [] + for idx, row in enumerate(reader): + if not idx: # first row + try: + assert(len(row) >= len(cols)) + except AssertionError: + return (0, 0, _(u"Invalid CSV format")) + continue + if len(row) < len(cols): + continue + pk, name, cats, state = row[0], row[1], row[2], row[3] + description, geom = row[4], row[5] + COL_INDEX = 6 + dct = {'description':description, + 'name':name, + 'origin':self.importer_instance.origin, + 'license':self.importer_instance.license} + cls = None + if 'POINT' in geom: + cls = Marker + dct['point'] = geom + elif 'LINE' in geom: + cls = Route + dct['route'] = geom + else: + continue + import_key = pk if pk else name + item, updated, created = self.create_or_update_item(cls, dct, + import_key, pk=pk) + if updated: + updated_item += 1 + if created: + new_item += 1 + for idx, col in enumerate(cols[COL_INDEX:]): + name, getter, setter_val = col + setter = getattr(item, setter_val) + val = row[idx+COL_INDEX] + setter(item, val) + return (new_item, updated_item, msg) @classmethod def export(cls, queryset): dct = {'description':unicode(datetime.date.today()), 'data':[]} cls_name = queryset.model.__name__.lower() - cols = cls.COLS[cls_name][:] + cols = cls.COLS for pm in queryset.model.properties(): - cols.append((pm.name, pm.getAttrName())) - header = [lbl for lbl, attr in cols] + cols.append((pm.name, pm.getAttrName(), pm.getAttrName()+'_set')) + header = [col[0] for col in cols] dct['data'].append(header) for item in queryset.all(): data = [] - for lbl, attr in cols: + for (lbl, attr, setr) in cols: if callable(attr): data.append(attr(item)) else: @@ -468,9 +538,66 @@ class CSVManager(ImportManager): dct['data'].append(data) filename = unicode_normalize(settings.PROJECT_NAME + dct['description']\ + '.csv') - result = render_to_response('chimere/export_%s.csv' % cls_name, dct) + result = render_to_response('chimere/export.csv', dct) return filename, result +class GeoRSSManager(ImportManager): + u""" + RSS importer. + This manager only gets and do not produce GeoRSSFeed + """ + + def get(self): + u""" + Get data from a GeoRSS simple source + + Return a tuple with: + - number of new item ; + - number of item updated ; + - error detail on error + """ + from models import Marker + new_item, updated_item, msg = 0, 0, '' + feed = feedparser.parse(self.importer_instance.source) + if feed['bozo']: + return (0, 0, _(u"RSS feed is not well formed")) + for item in feed['items']: + if "georss_point" not in item and 'georss_line' not in item: + continue + cls = None + dct = {'origin':self.importer_instance.origin, + 'license':self.importer_instance.license} + if 'georss_point' in item: + cls = Marker + try: + y, x = item['georss_point'].split(' ') + except ValueError: + continue + dct['point'] = 'SRID=4326;POINT(%s %s)' % (x, y) + for k in ['description', 'summary', 'value']: + if k in item: + dct['description'] = item[k] + break + else: + cls = Route + points = item['georss_line'].split(' ') + reordered_points = [] + # lat, lon -> x, y + for idx in xrange(len(points)/2): + reordered_points.append("%s %s" % (points[idx*2+1], + points[idx*2])) + dct['route'] = 'SRID=4326;LINESTRING(%s)' % \ + ",".join(reordered_points) + + dct['name'] = item['title'] + pl_id = item['id'] if 'id' in item else item['title'] + it, updated, created = self.create_or_update_item(cls, dct, pl_id) + if updated: + updated_item += 1 + if created: + new_item += 1 + return (new_item, updated_item, msg) + RE_HOOK = re.compile('\[([^\]]*)\]') # TODO: manage deleted item from OSM @@ -483,19 +610,16 @@ class OSMManager(ImportManager): """ default_source = settings.CHIMERE_XAPI_URL - def get(self, source=None): + def get(self): u""" Get data from the source - Args: - - source (None): input file if not provided get it from the distant - source provided in the importer instance. Return a tuple with: - new items; - updated items; - error detail on error. """ - source, msg = self.get_source_file(source, ['.osm'], + source, msg = self.get_source_file(['.osm'], extra_url=self.importer_instance.filtr) if not source: return (0, 0, msg) @@ -538,6 +662,10 @@ class OSMManager(ImportManager): for point_id in points if point_id in nodes]) dct = {'route':wkt, 'name':name, + 'origin':self.importer_instance.origin \ + or u'OpenStreetMap.org', + 'license':self.importer_instance.license \ + or u'ODbL', 'import_version':version} item, updated, created = self.create_or_update_item( Route, dct, node_id, version) @@ -567,6 +695,10 @@ class OSMManager(ImportManager): node.get('lat')) dct = {'point':point, 'name':name, + 'origin':self.importer_instance.origin \ + or u'OpenStreetMap.org', + 'license':self.importer_instance.license \ + or u'ODbL', 'import_version':version} item, updated, created = self.create_or_update_item( Marker, dct, node_id, version) |