summaryrefslogtreecommitdiff
path: root/chimere/utils.py
diff options
context:
space:
mode:
Diffstat (limited to 'chimere/utils.py')
-rw-r--r--chimere/utils.py221
1 files changed, 203 insertions, 18 deletions
diff --git a/chimere/utils.py b/chimere/utils.py
index d61ad19..a24e22f 100644
--- a/chimere/utils.py
+++ b/chimere/utils.py
@@ -21,6 +21,7 @@
Utilitaries
"""
+import csv
import datetime
import os
import re
@@ -429,9 +430,161 @@ class ShapefileManager(ImportManager):
buff.close()
return filename, zip_stream
-RE_NODE = re.compile('node\[([^\]]*)\]')
+class ImportManager:
+ u"""
+ Generic class for specific importers
+ """
+ default_source = None
+ def __init__(self, importer_instance):
+ self.importer_instance = importer_instance
+ self.default_name = " - ".join([cat.name
+ for cat in self.importer_instance.categories.order_by('name').all()])
+
+ def get(self):
+ pass
+
+ def put(self):
+ pass
+
+ def create_or_update_item(self, cls, values, import_key, version=None):
+ updated, created, item = False, False, None
+ if import_key:
+ dct_import = {
+ 'import_key__icontains':'%s:%s;' % (
+ self.importer_instance.importer_type,
+ import_key),
+ 'import_source':self.importer_instance.source}
+ try:
+ item = cls.objects.get(**dct_import)
+ if version and item.import_version == int(version):
+ # no update since the last import
+ return item, None, None
+ for k in values:
+ setattr(item, k, values[k])
+ try:
+ item.save()
+ except TypeError:
+ # error on data source
+ return None, False, False
+ updated = True
+ except ObjectDoesNotExist:
+ pass
+ if not item:
+ values.update({
+ 'import_source':self.importer_instance.source})
+ values['status'] = 'I'
+ try:
+ item = cls.objects.create(**values)
+ except TypeError:
+ # error on data source
+ return None, False, False
+ created = True
+ if import_key:
+ item.set_key(self.importer_instance.importer_type,
+ import_key)
+ item.categories.clear()
+ for cat in self.importer_instance.categories.all():
+ item.categories.add(cat)
+ return item, updated, created
+
+ @classmethod
+ def get_files_inside_zip(cls, zippedfile, suffixes, dest_dir=None):
+ try:
+ flz = zipfile.ZipFile(zippedfile)
+ except zipfile.BadZipfile:
+ return [], _(u"Bad zip file")
+ namelist = flz.namelist()
+ filenames = []
+ for suffix in suffixes:
+ current_file_name = None
+ for name in namelist:
+ if name.endswith(suffix) \
+ or name.endswith(suffix.lower()) \
+ or name.endswith(suffix.upper()):
+ current_file_name = name
+ filenames.append(current_file_name)
+ files = []
+ for filename in filenames:
+ if filename:
+ if dest_dir:
+ files.append(filename)
+ flz.extract(filename, dest_dir)
+ else:
+ files.append(flz.open(filename))
+ else:
+ files.append(None)
+ return files
-# manage deleted item from OSM
+ def get_source_file(self, source, suffixes, dest_dir=None,
+ extra_url=None):
+ if not hasattr(source, 'read'):
+ if not source:
+ source = self.importer_instance.source \
+ if self.importer_instance.source else self.default_source
+ try:
+ url = source
+ if extra_url:
+ url += extra_url
+ remotehandle = urllib2.urlopen(url)
+ source = StringIO.StringIO(remotehandle.read())
+ remotehandle.close()
+ except ValueError:
+ # assume it is a local file
+ try:
+ source = open(source)
+ except IOError, msg:
+ return (None, msg)
+ except urllib2.URLError as error:
+ return (None, error.message)
+ if self.importer_instance.zipped:
+ try:
+ files = self.get_files_inside_zip(source, suffixes, dest_dir)
+ except zipfile.BadZipfile:
+ return (None, _(u"Bad zip file"))
+ if not files or None in files:
+ return (None,
+ _(u"Missing file(s) inside the zip file"))
+ source = files[0] if len(suffixes) == 1 else files
+ return (source, None)
+
+class CSVManager(ImportManager):
+ u"""
+ CSV importer
+ """
+ _COLS = [("Id", 'pk'), (_(u"Name"), 'name'),
+ (_(u"Categories"), lambda obj:", ".join(
+ [c.name for c in obj.categories.all()])),
+ (_(u"State"), 'status')]
+ COLS = {'marker':_COLS+[(_(u"Description"), 'description'),
+ (_(u"Localisation"), lambda obj: obj.point.wkt)],
+ 'route':_COLS+[(_(u"Localisation"), lambda obj: obj.route.wkt)]}
+
+ @classmethod
+ def export(cls, queryset):
+ dct = {'description':unicode(datetime.date.today()), 'data':[]}
+ cls_name = queryset.model.__name__.lower()
+ cols = cls.COLS[cls_name][:]
+ for pm in queryset.model.properties():
+ cols.append((pm.name, pm.getAttrName()))
+ header = [lbl for lbl, attr in cols]
+ dct['data'].append(header)
+ for item in queryset.all():
+ data = []
+ for lbl, attr in cols:
+ if callable(attr):
+ data.append(attr(item))
+ print lbl, attr(item)
+ else:
+ data.append(getattr(item, attr))
+ dct['data'].append(data)
+ filename = unicode_normalize(settings.PROJECT_NAME + dct['description']\
+ + '.csv')
+ result = render_to_response('chimere/export_%s.csv' % cls_name, dct)
+ return filename, result
+
+RE_HOOK = re.compile('\[([^\]]*)\]')
+
+# TODO: manage deleted item from OSM
class OSMManager(ImportManager):
u"""
@@ -554,26 +707,58 @@ class OSMManager(ImportManager):
password=settings.CHIMERE_OSM_PASSWORD)
api.ChangesetCreate({u"comment": u"Import from Chimère %s" % \
get_version()})
- tag = RE_NODE.finddall(self.importer_instance.filtr)
- if not tag:
+ hooks = RE_HOOK.findall(self.importer_instance.filtr)
+ if not hooks:
return 0, _(u"Bad param")
- tag = tag[0].split('=')
- default_dct = {'tag':{tag[0]:tag[1]},
+ tags = {}
+ bbox = []
+ for hook in hooks:
+ key, value = hook.split('=')
+ if '*' in value or '|' in key or '|' in value:
+ continue
+ if key == 'bbox':
+ x1, y1, x2, y2 = [float(val) for val in value.split(',')]
+ bbox = GEOSGeometry(
+ 'POLYGON((%f %f,%f %f,%f %f,%f %f,%f %f))' % (
+ x1, y1, x2, y1, x2, y2, x1, y2, x1, y1), srid=4326)
+ continue
+ tags[key] = value
+ if not tags:
+ return 0, _(u"No non ambigious tag is defined in the XAPI request")
+ if not bbox:
+ return 0, _(u"No bounding box is defined in the XAPI request."\
+ u"If you are sure to manage the entire planet set the bounding box"\
+ u" to -180,-90,180,90")
+ default_dct = {'tag':tags,
'import_source':self.importer_instance.source}
- for idx, item in Marker.objects.filter(status='A',
- categories=self.importer_instance.categories.all()):
- dct = default_dct.update({
- 'name':item.name,
- 'lon':item.point.lon,
- 'lat':item.point.lat})
+ idx = -1
+ for idx, item in enumerate(Marker.objects.filter(status='A',
+ point__contained=bbox,
+ categories=self.importer_instance.categories.all(),
+ not_for_osm=False, modified_since_import=True).all()):
+ dct = default_dct.copy()
+ dct.update({'lon':item.point.x,
+ 'lat':item.point.y})
+ dct['tag']['name'] = item.name
node = None
- import_key = marker.get_key('OSM')
- if not import_key:
- node = OsmApi.NodeCreate(dct)
+ import_key = item.get_key('OSM')
+ updated = False
+ if import_key:
+ try:
+ dct['id'] = import_key
+ dct['version'] = item.import_version
+ node = api.NodeUpdate(dct)
+ updated = True
+ except OsmApi.ApiError, error:
+ if error.status == 404:
+ dct.pop('id')
+ dct.pop('version')
+ pass # if the node doesn't exist it is created
+ else:
+ raise
+ if not updated:
+ node = api.NodeCreate(dct)
item.set_key('OSM', node['id'])
- else:
- dct['id'] = import_key
- node = OsmApi.NodeUpdate(dct)
item.import_version = node['version']
item.save()
api.ChangesetClose()
generated by cgit v1.2.3 (git 2.39.1) at 2025-11-11 05:49:22 +0000