diff options
Diffstat (limited to 'chimere/utils.py')
| -rw-r--r-- | chimere/utils.py | 106 |
1 files changed, 85 insertions, 21 deletions
diff --git a/chimere/utils.py b/chimere/utils.py index 62d3786..b2c8ccc 100644 --- a/chimere/utils.py +++ b/chimere/utils.py @@ -21,10 +21,14 @@ Utilitaries """ -import urllib2 +import urllib2, re +from external_utils import OsmApi from lxml import etree +from chimere import get_version +from django.conf import settings from django.core.exceptions import ObjectDoesNotExist +from django.utils.translation import ugettext_lazy as _ class ImportManager: u""" @@ -42,11 +46,10 @@ class ImportManager: class KMLManager(ImportManager): u""" KML importer - The filtr argument has to be defined as a correct Xpath string pointing - to Placemark nodes. - A typical XPath string looks like: - "//kml:Folder/kml:name[text()='Subcategory 1']/../kml:Placemark" + The filtr argument has to be defined as the exact name of the folder to be + imported """ + XPATH = '//kml:Folder/kml:name[text()="%s"]/../kml:Placemark' def __init__(self, importer_instance, ns=''): self.importer_instance = importer_instance self.ns = ns @@ -62,11 +65,11 @@ class KMLManager(ImportManager): from models import Marker new_item, updated_item, msg = 0, 0, '' try: - source = urllib2.urlopen(self.importer_instance.source_url) + source = urllib2.urlopen(self.importer_instance.source) except ValueError: # assume it is a local file try: - source = open(self.importer_instance.source_url) + source = open(self.importer_instance.source) except IOError, msg: return (new_item, updated_item, msg) except urllib2.URLError as error: @@ -75,10 +78,11 @@ class KMLManager(ImportManager): # try to get default namespace if not self.ns: self.ns = tree.getroot().nsmap[None] - for placemark in tree.xpath(self.importer_instance.filtr, + for placemark in tree.xpath(self.XPATH % self.importer_instance.filtr, namespaces={'kml':self.ns}): name, point, linestring = None, None, None pl_id = placemark.attrib.get('id') + pl_key = 'kml-%d' % self.importer_instance.pk ns = '{%s}' % self.ns for item in placemark: if item.tag == ns + 'name': @@ -97,8 +101,8 @@ class KMLManager(ImportManager): m = None if pl_id: dct_import = { - 'import_key':pl_id, - 'import_source':self.importer_instance.source_url} + 'import_key__icontains':'%s:%s;' % (pl_key, pl_id), + 'import_source':self.importer_instance.source} try: m = Marker.objects.get(**dct_import) for k in dct: @@ -107,16 +111,23 @@ class KMLManager(ImportManager): updated_item += 1 except ObjectDoesNotExist: m = None - dct.update(dct_import) + dct.update({ + 'import_source':self.importer_instance.source}) if not m: dct['status'] = 'I' m = Marker.objects.create(**dct) new_item += 1 + if pl_id: + m.set_key(pl_key, pl_id) m.categories.clear() for cat in self.importer_instance.categories.all(): m.categories.add(cat) return (new_item, updated_item, msg) +RE_NODE = re.compile('node\[([^\]]*)\]') + +# manage deleted item from OSM + class OSMManager(ImportManager): u""" OSM importer @@ -124,23 +135,25 @@ class OSMManager(ImportManager): The filtr argument is XAPI args or empty if it is an OSM file. """ - def get(self): + def get(self, get_items=None): u""" Get data from the source Return a tuple with: - - number of new item ; - - number of item updated ; + - new items or number of new items if nb_only = True; + - updated items or number of updated items if nb_only = True; - error detail on error """ from models import Marker - new_item, updated_item, msg = 0, 0, '' + new_item, updated_item = 0 , 0 + items = [] + msg = '' try: - source = urllib2.urlopen(self.importer_instance.source_url+ + source = urllib2.urlopen(settings.CHIMERE_XAPI_URL+ self.importer_instance.filtr) except ValueError: # assume it is a local file try: - source = open(self.importer_instance.source_url) + source = open(self.importer_instance.source) except IOError, msg: return (new_item, updated_item, msg) except urllib2.URLError as error: @@ -163,10 +176,11 @@ class OSMManager(ImportManager): m = None if node_id: dct_import = { - 'import_key':node_id, - 'import_source':self.importer_instance.source_url} + 'import_key__icontains':'OSM:%s;' % (node_id), + 'import_source':self.importer_instance.source} try: m = Marker.objects.get(**dct_import) + items.append(m) if version and m.import_version == int(version): # no update since the last import continue @@ -176,12 +190,62 @@ class OSMManager(ImportManager): updated_item += 1 except ObjectDoesNotExist: m = None - dct.update(dct_import) + dct.update({ + 'import_source':self.importer_instance.source}) if not m: dct['status'] = 'I' m = Marker.objects.create(**dct) new_item += 1 + items.append(m) + if node_id: + m.set_key('OSM', node_id) m.categories.clear() for cat in self.importer_instance.categories.all(): m.categories.add(cat) - return (new_item, updated_item, msg) + if not get_items: + return (new_item, updated_item, msg) + return (items, new_item, updated_item, msg) + + def put(self): + # first of all: reimport in order to verify that no changes has been + # made since the last import + from models import Marker + items, new_item, updated_item, msg = self.get(get_items=True) + # check if import is possible + if msg: + return 0, msg + if new_item: + return 0, _(u"New items imported - validate them before exporting") + if Marker.objects.filter(status='I').count(): + return 0, _(u"There are items from a former import not yet " + u"validated - validate them before exporting") + # start import + api = OsmApi.OsmApi(api=settings.CHIMERE_OSM_API_URL, + username=settings.CHIMERE_OSM_USER, + password=settings.CHIMERE_OSM_PASSWORD) + api.ChangesetCreate({u"comment": u"Import from Chimère %s" % \ + get_version()}) + tag = RE_NODE.finddall(self.importer_instance.filtr) + if not tag: + return 0, _(u"Bad param") + tag = tag[0].split('=') + default_dct = {'tag':{tag[0]:tag[1]}, + 'import_source':self.importer_instance.source} + for idx, item in Marker.objects.filter(status='A', + categories=self.importer_instance.categories.all()): + dct = default_dct.update({ + 'name':item.name, + 'lon':item.point.lon, + 'lat':item.point.lat}) + node = None + import_key = marker.get_key('OSM') + if not import_key: + node = OsmApi.NodeCreate(dct) + item.set_key('OSM', node['id']) + else: + dct['id'] = import_key + node = OsmApi.NodeUpdate(dct) + item.import_version = node['version'] + item.save() + api.ChangesetClose() + return idx+1, None |