path: root/chimere/utils.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2012-2015  Étienne Loks  <etienne.loks_AT_peacefrogsDOTnet>

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# See the file COPYING for details.

"""
Utilitaries
"""

import csv
import collections
import datetime
import feedparser
import json
import os
import re
import StringIO
import tempfile
import urllib2
import unicodedata
import zipfile

from osgeo import ogr, osr
from lxml import etree

from django.conf import settings
from django.contrib.gis.gdal import DataSource, OGRGeomType, check_err
from django.contrib.gis.geos import GEOSGeometry
from django.core.exceptions import ObjectDoesNotExist
from django.shortcuts import render_to_response
from django.utils.translation import ugettext_lazy as _

from chimere import get_version
from external_utils import OsmApi


def unicode_normalize(string):
    if type(string) == str:
        string = unicode(string.decode('utf-8'))
    return ''.join(
        (c for c in unicodedata.normalize('NFD', string)
         if unicodedata.category(c) not in ('Mn', 'Sm', 'Sc')))


class ImportManager(object):
    u"""
    Generic class for specific importers
    """
    default_source = None

    def __init__(self, importer_instance):
        self.importer_instance = importer_instance
        if self.importer_instance.default_name:
            self.default_name = self.importer_instance.default_name
        else:
            self.default_name = " - ".join([
                cat.name
                for cat in self.importer_instance.categories.order_by(
                    'name').all()])

    def get(self):
        raise NotImplementedError

    def put(self, extra_args={}):
        raise NotImplementedError

    def create_or_update_item(self, cls, values, import_key, version=None,
                              key='', pk=None, category=None):
        updated, created, item = False, False, None
        import_key = unicode(import_key).replace(':', '^')
        if not values.get('name'):
            values['name'] = self.default_name
        if not key:
            key = self.importer_instance.importer_type
        item = None
        if import_key or pk:
            dct_import = {
                'import_key__icontains': '%s:%s;' % (key, import_key),
                'import_source': self.importer_instance.source}
            ref_item = cls.objects.filter(**dct_import)
            try:
                item = None
                if pk:
                    ref_item = cls.objects.get(pk=pk)
                else:
                    ref_item = cls.objects.filter(**dct_import)
                    if not ref_item.count():
                        raise ObjectDoesNotExist
                    ref_item = ref_item.all()[0]
                if version and ref_item.import_version == int(version):
                    # no update since the last import
                    return ref_item, None, None
                if not self.importer_instance.overwrite \
                   and ref_item.modified_since_import:
                    return ref_item, None, None
                else:
                    item = ref_item
                    for k in values:
                        if values[k]:
                            setattr(item, k, values[k])
                    try:
                        item.save()
                        # force the modified_since_import status
                        item.modified_since_import = False
                        item.save()
                    except TypeError:
                        # error on data source
                        return None, False, False
                    updated = True
            except ObjectDoesNotExist:
                pass
        if not item:
            if not self.importer_instance.get_description and \
               self.importer_instance.default_description:
                values['description'] = \
                    self.importer_instance.default_description
            values.update({
                'import_source': self.importer_instance.source})
            values['status'] = self.importer_instance.default_status
            if not self.importer_instance.associate_marker_to_way\
                    and cls.__name__ == 'Route':
                values['has_associated_marker'] = False
            try:
                item = cls.objects.create(**values)
                item.modified_since_import = False
                item.save()
            except TypeError:
                # error on data source
                return None, False, False
            created = True
        if import_key:
            item.set_key(key, import_key)
        item.categories.clear()
        if category:
            item.categories.add(category)
        else:
            for cat in self.importer_instance.categories.all():
                item.categories.add(cat)
        return item, updated, created

    @classmethod
    def get_files_inside_zip(cls, zippedfile, suffixes, dest_dir=None):
        try:
            flz = zipfile.ZipFile(zippedfile)
        except zipfile.BadZipfile:
            return [], _(u"Bad zip file")
        namelist = flz.namelist()
        filenames = []
        for suffix in suffixes:
            current_file_name = None
            for name in namelist:
                if name.endswith(suffix) \
                        or name.endswith(suffix.lower()) \
                        or name.endswith(suffix.upper()):
                    current_file_name = name
            filenames.append(current_file_name)
        files = []
        for filename in filenames:
            if filename:
                if dest_dir:
                    files.append(filename)
                    flz.extract(filename, dest_dir)
                else:
                    files.append(flz.open(filename))
            else:
                files.append(None)
        return files

    def get_source_file(self, suffixes, dest_dir=None,
                        extra_url=None):
        source = self.importer_instance.source_file
        if not hasattr(source, 'read'):
            if not source:
                source = self.importer_instance.source \
                    if self.importer_instance.source else self.default_source
            try:
                url = source
                if extra_url:
                    url += extra_url
                remotehandle = urllib2.urlopen(url)
                source = StringIO.StringIO(remotehandle.read())
                remotehandle.close()
            except ValueError:
                # assume it is a local file
                try:
                    source = open(source)
                except IOError, msg:
                    return (None, msg)
            except (urllib2.URLError, AttributeError) as error:
                return (None, error.message)
        if self.importer_instance.zipped:
            try:
                files = self.get_files_inside_zip(source, suffixes, dest_dir)
            except zipfile.BadZipfile:
                return (None, _(u"Bad zip file"))
            if not files or None in files:
                return (None,
                        _(u"Missing file(s) inside the zip file"))
            source = files[0] if len(suffixes) == 1 else files
        return (source, None)


class KMLManager(ImportManager):
    u"""
    KML importer
    The filtr argument has to be defined as the exact name of the folder to be
    imported
    """
    XPATH = '//kml:Folder/kml:name[text()="%s"]/../kml:Placemark'
    DEFAULT_XPATH = '//kml:Placemark'

    def __init__(self, importer_instance, ns=''):
        super(KMLManager, self).__init__(importer_instance)
        self.ns = ns

    def get(self):
        u"""
        Get data from a KML source

        Return a tuple with:
         - number of new item ;
         - number of item updated ;
         - error detail on error
        """
        from models import Marker, Route
        new_item, updated_item, msg = 0, 0, ''
        source, msg = self.get_source_file(['.kml'])
        if msg:
            return (0, 0, msg)
        doc = source
        # remove empty lines before declaration (bad XML file)
        if hasattr(source, 'getvalue'):
            splitted = source.getvalue().split('\n')
            for idx, line in enumerate(splitted):
                if line.strip():
                    break
            doc = StringIO.StringIO("\n".join(splitted[idx:]))
        try:
            tree = etree.parse(doc)
        except:
            return (0, 0, _(u"Bad XML file"))
        # try to get default namespace
        if not self.ns:
            self.ns = tree.getroot().nsmap[None]
        xpath = self.XPATH % self.importer_instance.filtr \
            if self.importer_instance.filtr else self.DEFAULT_XPATH
        for placemark in tree.xpath(xpath,
                                    namespaces={'kml': self.ns}):
            name, point, line = None, None, None
            pl_id = placemark.attrib.get('id')
            pl_key = 'kml-%d' % self.importer_instance.pk
            ns = '{%s}' % self.ns
            description = ''
            for item in placemark:
                if item.tag == ns + 'name':
                    name = item.text
                    if not pl_id:
                        # if no ID is provided assume that name is a key
                        pl_id = name
                elif item.tag == ns + 'description':
                    if self.importer_instance.get_description:
                        description = item.text
                elif item.tag == ns + 'Point':
                    for coord in item:
                        if coord.tag == ns + 'coordinates':
                            x, y, z = coord.text.split(',')
                            point = 'SRID=4326;POINT(%s %s)' % (x, y)
                elif item.tag == ns + 'LineString':
                    for coord in item:
                        if coord.tag == ns + 'coordinates':
                            points = coord.text.replace('\n', ' ').split(' ')
                            points = ",".join([" ".join(p.split(',')[:2])
                                               for p in points if p])
                            line = 'SRID=4326;LINESTRING(%s)' % points
            cls = None
            dct = {'description': description,
                   'name': name,
                   'origin': self.importer_instance.origin,
                   'license': self.importer_instance.license}
            if point:
                dct['point'] = point
                cls = Marker
            if line:
                dct['route'] = line
                dct.pop('description')
                cls = Route
            if cls:
                item, updated, created = self.create_or_update_item(
                    cls, dct, pl_id, key=pl_key)
                if updated:
                    updated_item += 1
                if created:
                    new_item += 1
        return (new_item, updated_item, msg)

    @classmethod
    def export(cls, queryset):
        dct = {
            'name': settings.PROJECT_NAME,
            'description': unicode(datetime.date.today()),
            'locations': queryset.all()
        }
        filename = unicode_normalize(settings.PROJECT_NAME + dct['description']
                                     + '.kml')
        result = render_to_response('chimere/export.kml', dct)
        return filename, result


class ShapefileManager(ImportManager):
    u"""
    Shapefile importer
    """
    def get(self):
        u"""
        Get data from a Shapefile source

        Return a tuple with:
         - number of new item ;
         - number of item updated ;
         - error detail on error
        """
        from models import Marker, Route
        new_item, updated_item, msg = 0, 0, ''
        tmpdir = tempfile.mkdtemp()
        sources, msg = self.get_source_file(['.shp', '.dbf', '.prj', '.shx'],
                                            dest_dir=tmpdir)
        if msg:
            return (0, 0, msg)
        if not sources:
            return (0, 0, _(u"Error while reading the data source."))
        # get the srid
        srid = self.importer_instance.srid
        if not srid:
            prjfilename = tmpdir + os.sep + sources[2]
            try:
                from osgeo import osr
                with open(prjfilename, 'r') as prj_file:
                    prj_txt = prj_file.read()
                    srs = osr.SpatialReference()
                    srs.ImportFromESRI([prj_txt])
                    srs.AutoIdentifyEPSG()
                    srid = srs.GetAuthorityCode(None)
            except ImportError:
                pass
            if not srid:
                # try with the default projection
                srid = settings.CHIMERE_EPSG_DISPLAY_PROJECTION
                msg = _(u"SRID cannot be guessed. The default SRID (%s) has "
                        u"been used.") % srid
                # If imported items are not well located "
                #        u"ask your data provider for the SRID to use.") % srid
        shapefilename = tmpdir + os.sep + sources[0]
        ds = DataSource(shapefilename)
        lyr = ds[0]
        # for this first version it is assumed that the first field is a
        # id name and the second field is the name
        id_name = lyr.fields[0] if len(lyr.fields) > 0 else None
        # test if id_name is well guess
        if id_name:
            ids = lyr.get_fields(id_name)
            if len(ids) != len(set(ids)):
                id_name = None
        lbl_name = None
        if len(lyr.fields) > 1:
            lbl_name = lyr.fields[1]
        elif id_name:
            lbl_name = id_name
        if lyr.geom_type not in ('Point', 'LineString'):
            return (0, 0, _(u"Type of geographic item (%s) of this shapefile "
                            u"is not managed by Chimère.") % lyr.geom_type)
        geom_key = 'point' if lyr.geom_type == 'Point' else 'route'
        geom_cls = Marker if lyr.geom_type == 'Point' else Route
        # indexes = []
        for idx, feat in enumerate(lyr):
            name = unicode(idx)
            if lbl_name:
                name = feat.get(lbl_name)
                try:
                    name = unicode(name)
                except UnicodeDecodeError:
                    try:
                        name = unicode(
                            name.decode(settings.CHIMERE_SHAPEFILE_ENCODING))
                    except:
                        continue
            try:
                geoms = [feat.geom.wkt]
            except:
                return (0, 0, _(u"Bad Shapefile"))
            if feat.geom.geom_type == 'MultiLineString':
                geoms = [geom.wkt for geom in feat.geom]
            import_key = feat.get(id_name) if id_name and len(geoms) == 1 \
                else ''
            for geom in geoms:
                dct = {
                    geom_key: 'SRID=%s;%s' % (srid, geom),
                    'name': name,
                    'origin': self.importer_instance.origin,
                    'license': self.importer_instance.license
                }
                item, updated, created = self.create_or_update_item(
                    geom_cls, dct, import_key)
                if updated:
                    updated_item += 1
                if created:
                    new_item += 1
        # clean up
        tmpdirs = set()
        for src in sources:
            dirs = os.sep.join(src.split(os.sep)[:-1])
            if dirs:
                tmpdirs.add(tmpdir + os.sep + dirs)
            os.remove(tmpdir + os.sep + src)
        for dr in tmpdirs:
            os.removedirs(dr)
        return (new_item, updated_item, msg)

    @classmethod
    def export(cls, queryset):
        date = unicode(datetime.date.today())

        tmp = tempfile.NamedTemporaryFile(suffix='.shp', mode='w+b')
        tmp.close()

        tmp_name = tmp.name
        field_names = [field.name for field in queryset.model._meta.fields]
        geo_field = getattr(
            queryset.model,
            'point' if 'point' in field_names else 'route')._field

        dr = ogr.GetDriverByName('ESRI Shapefile')
        ds = dr.CreateDataSource(tmp_name)
        if ds is None:
            raise Exception(_(u'Could not create file!'))
        ogr_type = OGRGeomType(geo_field.geom_type).num
        srs = osr.SpatialReference()
        srs.ImportFromEPSG(geo_field.srid)

        layer = ds.CreateLayer('lyr', srs=srs, geom_type=ogr_type)

        for field_name in ('name', 'category'):
            field_defn = ogr.FieldDefn(str(field_name), ogr.OFTString)
            field_defn.SetWidth(255)
            if layer.CreateField(field_defn) != 0:
                raise Exception(_(u'Failed to create field'))

        feature_def = layer.GetLayerDefn()

        for item in queryset:
            # duplicate items when in several categories
            for category in item.categories.all():
                feat = ogr.Feature(feature_def)
                feat.SetField('name', str(unicode_normalize(item.name)[:80]))
                feat.SetField('category',
                              str(unicode_normalize(category.name)[:80]))

                geom = getattr(item, geo_field.name)
                if not geom:
                    continue
                ogr_geom = ogr.CreateGeometryFromWkt(geom.wkt)
                check_err(feat.SetGeometry(ogr_geom))
            check_err(layer.CreateFeature(feat))
        # Cleaning up
        ds.Destroy()

        # writing to a zip file
        filename = unicode_normalize(settings.PROJECT_NAME) + '-' + date
        buff = StringIO.StringIO()
        zip_file = zipfile.ZipFile(buff, 'w', zipfile.ZIP_DEFLATED)
        suffixes = ['shp', 'shx', 'prj', 'dbf']
        for suffix in suffixes:
            name = tmp_name.replace('.shp', '.' + suffix)
            arcname = '.'.join((filename, suffix))
            zip_file.write(name, arcname=arcname)
        zip_file.close()
        buff.flush()
        zip_stream = buff.getvalue()
        buff.close()
        return filename, zip_stream


class CSVManager(ImportManager):
    u"""
    CSV importer
    """
    @classmethod
    def set_categories(value):
        return

    # (label, getter, setter)
    COLS = [("Id", 'pk', 'pk'), (_(u"Name"), 'name', 'name'),
            (_(u"Categories"), lambda obj: ", ".join(
                [c.name for c in obj.categories.all()]), set_categories),
            (_(u"State"), 'status', lambda x: x),
            (_(u"Description"), 'description', 'description'),
            (_(u"Localisation"), 'geometry', 'geometry')]

    def get(self):
        u"""
        Get data from a CSV source

        Return a tuple with:
         - number of new item ;
         - number of item updated ;
         - error detail on error
        """
        from models import Marker, Route
        new_item, updated_item, msg = 0, 0, ''
        source, msg = self.get_source_file(['.csv'])
        if msg:
            return (0, 0, msg)
        reader = csv.reader(source, delimiter=';', quotechar='"')
        prop_cols = []
        for pm in Marker.all_properties():
            prop_cols.append((pm.name, pm.getAttrName(),
                              pm.getAttrName() + '_set'))
        cols = list(self.COLS) + prop_cols
        # datas = []
        for idx, row in enumerate(reader):
            if not idx:  # first row
                try:
                    assert(len(row) >= len(cols))
                except AssertionError:
                    return (0, 0, _(u"Invalid CSV format"))
                continue
            if len(row) < len(cols):
                continue
            # pk, name, cats, state = row[0], row[1], row[2], row[3]
            pk, name = row[0], row[1]
            geom = row[5]
            description = ''
            if self.importer_instance.get_description:
                description = row[4]
            COL_INDEX = 6
            dct = {'description': description,
                   'name': name,
                   'origin': self.importer_instance.origin,
                   'license': self.importer_instance.license}
            cls = None
            if 'POINT' in geom:
                cls = Marker
                dct['point'] = geom
            elif 'LINE' in geom:
                cls = Route
                dct['route'] = geom
            else:
                continue
            import_key = pk if pk else name.decode('utf-8')
            item, updated, created = self.create_or_update_item(
                cls, dct, import_key, pk=pk)
            if updated:
                updated_item += 1
            if created:
                new_item += 1
            for idx, col in enumerate(cols[COL_INDEX:]):
                name, getter, setter_val = col
                setter = getattr(item, setter_val)
                val = row[idx + COL_INDEX]
                setter(item, val)
        return (new_item, updated_item, msg)

    @classmethod
    def export(cls, queryset):
        dct = {'description': unicode(datetime.date.today()), 'data': []}
        # cls_name = queryset.model.__name__.lower()
        cols = list(cls.COLS)
        for pm in queryset.model.all_properties():
            cols.append((pm.name, pm.getAttrName(), pm.getAttrName() + '_set'))
        header = [col[0] for col in cols]
        dct['data'].append(header)
        for item in queryset.all():
            data = []
            for (lbl, attr, setr) in cols:
                if callable(attr):
                    data.append(attr(item))
                else:
                    data.append(getattr(item, attr))
            dct['data'].append(data)
        filename = unicode_normalize(settings.PROJECT_NAME + dct['description']
                                     + '.csv')
        result = render_to_response('chimere/export.csv', dct)
        return filename, result


class GeoRSSManager(ImportManager):
    u"""
    RSS importer.
    This manager only gets and do not produce GeoRSSFeed
    """

    def get(self):
        u"""
        Get data from a GeoRSS simple source

        Return a tuple with:
         - number of new item ;
         - number of item updated ;
         - error detail on error
        """
        from models import Marker, Route
        new_item, updated_item, msg = 0, 0, ''
        feed = feedparser.parse(self.importer_instance.source)
        if feed['bozo'] and not isinstance(
                feed['bozo_exception'], feedparser.CharacterEncodingOverride):
            return (0, 0, _(u"RSS feed is not well formed"))
        for item in feed['items']:
            if "georss_point" not in item and 'georss_line' not in item \
               and not ("geo_lat" in item and "geo_long" in item):
                continue
            cls = None
            dct = {'origin': self.importer_instance.origin,
                   'license': self.importer_instance.license}
            if 'georss_point' in item or "geo_lat" in item:
                cls = Marker
                if 'georss_point' in item:
                    try:
                        y, x = item['georss_point'].split(' ')
                    except ValueError:
                        continue
                else:
                    y = item['geo_lat']
                    x = item['geo_long']
                dct['point'] = 'SRID=4326;POINT(%s %s)' % (x, y)
                if self.importer_instance.get_description:
                    for k in ['description', 'summary', 'value']:
                        if k in item:
                            dct['description'] = item[k]
                            break
            else:
                cls = Route
                points = item['georss_line'].split(' ')
                reordered_points = []
                # lat, lon -> x, y
                for idx in xrange(len(points) / 2):
                    reordered_points.append("%s %s" % (points[idx * 2 + 1],
                                                       points[idx * 2]))
                dct['route'] = 'SRID=4326;LINESTRING(%s)' % \
                    ",".join(reordered_points)

            dct['name'] = item['title']
            pl_id = item['id'] if 'id' in item else item['title']
            it, updated, created = self.create_or_update_item(cls, dct, pl_id)
            if updated:
                updated_item += 1
            if created:
                new_item += 1
        return (new_item, updated_item, msg)


class JsonManager(ImportManager):
    u"""
    Json importer.
    This manager only gets and do not produce Json feed
    """

    def get(self):
        u"""
        Get data from a json simple source

        Return a tuple with:
         - number of new item ;
         - number of item updated ;
         - error detail on error
        """
        from models import Marker
        new_item, updated_item, msg = 0, 0, ''
        source, msg = self.get_source_file(['.json'])
        if msg:
            return (0, 0, msg)

        vals = source.read().replace('\n', ' ')
        try:
            values = json.JSONDecoder(
                object_pairs_hook=collections.OrderedDict).decode(vals)
        except ValueError as e:
            return (new_item, updated_item,
                    _(u"JSON file is not well formed: " + e.message))
        # configuration in filtr
        try:
            filtr = json.JSONDecoder().decode(self.importer_instance.filtr)
        except ValueError:
            return (
                new_item, updated_item,
                _(u"Bad configuration: filter field must be a valid "
                  u"JSON string"))

        vls = filtr.values()
        for k in ('name', 'id', 'description'):
            if k not in vls:
                return (
                    new_item, updated_item,
                    _(u"A key must be associated to \"%s\" in the "
                      u"filter.") % k)

        default_dct = {'origin': self.importer_instance.origin,
                       'license': self.importer_instance.license}
        if 'prefix_name' in filtr:
            default_dct['name'] = filtr.pop('prefix_name')
        if 'prefix_description' in filtr:
            default_dct['description'] = filtr.pop('prefix_description')
        if self.importer_instance.default_localisation:
            default_dct['point'] = self.importer_instance.default_localisation

        for item in values:
            dct = default_dct.copy()
            for k in filtr:
                if k.startswith('prefix_') or k.startswith('suffix_'):
                    continue
                if k in item and item[k]:
                    if filtr[k] not in dct:
                        dct[filtr[k]] = ""
                    else:
                        if filtr[k] == 'description':
                            dct[filtr[k]] += "<br/>"
                        else:
                            dct[filtr[k]] += " "
                    dct[filtr[k]] += item[k]
            if 'point' in item:
                x, y = item['point'].split(",")
                dct['point'] = 'SRID=4326;POINT(%s %s)' % (x, y)
            elif 'lat' in item and item['lat'] \
                    and 'lon' in item and item['lon']:
                dct['point'] = 'SRID=4326;POINT(%s %s)' % (item['lon'],
                                                           item['lat'])
            elif 'x' in item and item['x'] \
                    and 'y' in item and item['y']:
                dct['point'] = 'SRID=4326;POINT(%s %s)' % (item['x'],
                                                           item['y'])
            if not dct['point']:
                continue
            for k in filtr:
                if k.startswith('prefix_') or k.startswith('suffix_'):
                    pos = k.split('_')[0]
                    key = '_'.join(k.split('_')[1:])
                    if key in dct:
                        if pos == 'prefix':
                            dct[key] = filtr[k] + dct[key]
                        else:
                            dct[key] += filtr[k]
            cls = Marker
            pl_id = (dct.pop('id') if 'id' in dct else dct['name']) \
                + "-" + unicode(self.importer_instance.pk)
            it, updated, created = self.create_or_update_item(cls, dct, pl_id)
            if updated:
                updated_item += 1
            if created:
                new_item += 1
        return (new_item, updated_item, msg)

RE_HOOK = re.compile('\[([^\]]*)\]')

# TODO: manage deleted item from OSM


class OSMManager(ImportManager):
    u"""
    OSM importer/exporter
    The source url is a path to an OSM file or a XAPI url
    The filtr argument is XAPI args or empty if it is an OSM file.
    """
    default_source = settings.CHIMERE_XAPI_URL

    def get(self):
        u"""
        Get data from the source

        Return a tuple with:
        - new items;
        - updated items;
        - error detail on error.
        """
        source, msg = self.get_source_file(
            ['.osm'], extra_url=self.importer_instance.filtr)
        if not source:
            return (0, 0, msg)

        tree = etree.parse(source)
        # only import node or ways
        if tree.xpath('count(//way)') and tree.xpath('count(//node)'):
            return self.import_ways(tree)
        elif tree.xpath('count(//node)'):
            return self.import_nodes(tree)
        return 0, 0, _(u"Nothing to import")

    def import_ways(self, tree):
        from chimere.models import Route
        msg, items, new_item, updated_item = "", [], 0, 0
        nodes = {}
        for node in tree.xpath('//node'):
            node_id = node.attrib.get('id')
            for item in node:
                k = item.attrib.get('k')
            if node_id:
                nodes[node_id] = '%s %s' % (node.get('lon'),
                                            node.get('lat'))
        for way in tree.xpath('//way'):
            name = None
            points = []
            node_id = way.attrib.get('id')
            version = way.attrib.get('version')
            for item in way:
                k = item.attrib.get('k')
                if k == 'name':
                    name = item.attrib.get('v')
                if item.tag == 'nd':
                    points.append(item.get('ref'))
            if not points:
                continue
            wkt = 'SRID=4326;LINESTRING(%s)' % ",".join(
                [nodes[point_id] for point_id in points if point_id in nodes])
            dct = {'route': wkt,
                   'name': name,
                   'origin': self.importer_instance.origin
                   or u'OpenStreetMap.org',
                   'license': self.importer_instance.license
                   or u'ODbL',
                   'import_version': version}
            item, updated, created = self.create_or_update_item(
                Route, dct, node_id, version)
            if updated:
                updated_item += 1
            if created:
                new_item += 1
            items.append(item)
        return new_item, updated_item, msg

    def import_nodes(self, tree):
        from chimere.models import Marker
        msg, items, new_item, updated_item = "", [], 0, 0
        for node in tree.xpath('//node'):
            name = None
            node_id = node.attrib.get('id')
            if not node_id:
                continue
            version = node.attrib.get('version')
            for item in node:
                k = item.attrib.get('k')
                if k == 'name':
                    name = item.attrib.get('v')
            point = 'SRID=4326;POINT(%s %s)' % (node.get('lon'),
                                                node.get('lat'))
            dct = {'point': point,
                   'name': name,
                   'origin': self.importer_instance.origin
                   or u'OpenStreetMap.org',
                   'license': self.importer_instance.license
                   or u'ODbL',
                   'import_version': version}
            item, updated, created = self.create_or_update_item(
                Marker, dct, node_id, version)
            if updated:
                updated_item += 1
            if created:
                new_item += 1
            items.append(item)
        return (new_item, updated_item, msg)

    def put(self, extra_args={}):
        # first of all: reimport in order to verify that no changes has been
        # made since the last import
        from models import Marker
        new_item, updated_item, msg = self.get()
        # check if import is possible
        if msg:
            return 0, msg
        if new_item:
            return 0, _(u"New items imported - validate them before exporting")
        if Marker.objects.filter(status='I').count():
            return 0, _(u"There are items from a former import not yet "
                        u"validated - validate them before exporting")
        # start import
        api = settings.CHIMERE_OSM_API_URL
        username = settings.CHIMERE_OSM_USER
        password = settings.CHIMERE_OSM_PASSWORD
        if extra_args:
            try:
                api = extra_args['api']
                username = extra_args['username']
                password = extra_args['password']
            except KeyError:
                return 0, _(u"Bad params - programming error")
        username = username.encode('latin1')
        password = password.encode('latin1')
        api = OsmApi.OsmApi(api=api, username=username, password=password)
        api.ChangesetCreate({u"comment": u"Import from Chimère %s" %
                             get_version()})
        hooks = RE_HOOK.findall(self.importer_instance.filtr)
        if not hooks:
            hooks = RE_HOOK.findall(self.importer_instance.source)
            if not hooks:
                return 0, _(u"Bad param")
        tags = {}
        bbox = []
        for hook in hooks:
            key, value = hook.split('=')
            if '*' in value or '|' in key or '|' in value:
                continue
            if key == 'bbox':
                x1, y1, x2, y2 = [float(val) for val in value.split(',')]
                bbox = GEOSGeometry(
                    'POLYGON((%f %f,%f %f,%f %f,%f %f,%f %f))' % (
                        x1, y1, x2, y1, x2, y2, x1, y2, x1, y1), srid=4326)
                continue
            tags[key] = value
        if not tags:
            return 0, _(u"No non ambigious tag is defined in the XAPI request")
        if not bbox:
            return 0, _(
                u"No bounding box is defined in the XAPI request."
                u"If you are sure to manage the entire planet set the "
                u"bounding box to -180,-90,180,90")
        default_dct = {'tag': tags,
                       'import_source': self.importer_instance.source}
        idx = -1
        for idx, item in enumerate(
                Marker.objects.filter(
                    status='A',
                    point__contained=bbox,
                    categories=self.importer_instance.categories.all(),
                    not_for_osm=False, modified_since_import=True,
                    route=None).all()):
            dct = default_dct.copy()
            dct.update({'lon': item.point.x,
                        'lat': item.point.y})
            dct['tag']['name'] = item.name
            node = None
            import_key = item.get_key('OSM')
            updated = False
            if import_key:
                try:
                    dct['id'] = import_key
                    dct['version'] = item.import_version
                    node = api.NodeUpdate(dct)
                    updated = True
                except OsmApi.ApiError, error:
                    if error.status == 404:
                        dct.pop('id')
                        dct.pop('version')
                        pass  # if the node doesn't exist it is created
                    else:
                        raise
            if not updated:
                node = api.NodeCreate(dct)
                item.set_key('OSM', node['id'])
            item.import_version = node['version']
            item.save()
        api.ChangesetClose()
        return idx + 1, None


import chardet
import HTMLParser
from BeautifulSoup import BeautifulSoup


RE_CLEANS = ((re.compile('(\n)*|^( )*(\n)*( )*|( )*(\n)*( )*$'), ''),
             (re.compile(' ( )*'), ' '),
             (re.compile(r"""<a href=["'](?!https?)(.*)["']"""),
              '<a href="%(base_url)s\\1"'),
             )

from calendar import TimeEncoding, month_name


def get_month_name(month_no, locale):
    with TimeEncoding(locale) as encoding:
        s = month_name[month_no]
        if encoding is not None:
            s = s.decode(encoding)
        return s

MONTH_NAMES = {locale: [get_month_name(no_month, locale + '.UTF-8')
               for no_month in xrange(1, 13)] for locale in ['fr_FR']}

try:
    UNI_MONTH_NAMES = {locale: [m.decode('utf-8') for m in MONTH_NAMES[locale]]
                       for locale in MONTH_NAMES}
except UnicodeEncodeError:
    UNI_MONTH_NAMES = {locale: [m for m in MONTH_NAMES[locale]]
                       for locale in MONTH_NAMES}

DATE_PARSINGS = {
    'fr_FR': [
        re.compile(r'(?P<day1>\d{1,2}) '
                   r'(?P<month1>' + '|'.join(UNI_MONTH_NAMES['fr_FR']) + ') '
                   r'(?P<year1>\d{4})?[^\d]*'
                   r'(?P<day2>\d{1,2}) '
                   r'(?P<month2>' + '|'.join(UNI_MONTH_NAMES['fr_FR']) + ') *'
                   r'(?P<year2>\d{4})?.*'),
        re.compile(r'(?P<day1>\d{1,2}) '
                   r'(?P<month1>' + '|'.join(UNI_MONTH_NAMES['fr_FR']) + ') * '
                   r'(?P<year1>\d{4})?')],
    'en': [
        re.compile(r'(?P<year1>\d{4})-'
                   r'(?P<month1>\d{2})-'
                   r'(?P<day1>\d{2})'
                   r'(?:T'
                   r'(?P<hour1>\d{2})?:'
                   r'(?P<minut1>\d{2})?:'
                   r'(?P<second1>\d{2})'
                   r')?.*'
                   r'(?P<year2>\d{4})-'
                   r'(?P<month2>\d{2})-'
                   r'(?P<day2>\d{2})'
                   r'(?:T'
                   r'(?P<hour2>\d{2})?:'
                   r'(?P<minut2>\d{2})?:'
                   r'(?P<second2>\d{2})'
                   r')?.*'),
        re.compile(r'(?P<year1>\d{4})-'
                   r'(?P<month1>\d{2})-'
                   r'(?P<day1>\d{2})'
                   r'(?:T'
                   r'(?P<hour1>\d{2})?:'
                   r'(?P<minut1>\d{2})?:'
                   r'(?P<second1>\d{2})'
                   r')?')],
}


def clean_field(value):
    return value.strip()


class HtmlXsltManager(ImportManager):
    PARSER = 'HTMLParser'

    def get(self):
        u"""
        Get data from the source

        Return a tuple with:
        - new items;
        - updated items;
        - error detail on error.
        """
        from models import Marker
        self.marker_cls = Marker
        try:
            main_page = urllib2.urlopen(self.importer_instance.source)
            assert main_page.getcode() == 200
        except (urllib2.URLError, AssertionError):
            return (0, 0, _(u"Source page is unreachable."))
        data = main_page.read()
        encoding = chardet.detect(data)
        data = data.decode(encoding['encoding'])

        soup = BeautifulSoup(data)
        main_page = soup.prettify()
        # convert it to valid XHTML
        # doc, errors = tidy_document(main_page)
        doc = main_page
        dom = etree.HTML(doc, getattr(etree, self.PARSER)())
        try:
            xslt = etree.parse(self.importer_instance.source_file)
            self.importer_instance.source_file.seek(0)
            transform = etree.XSLT(xslt)
        except (etree.XSLTParseError, etree.XMLSyntaxError, TypeError):
            return (0, 0, _(u"The source file is not a valid XSLT file."))
        newdom = transform(dom)
        items = []
        # load an alternate xslt file to apply to linked page
        transform_child = None
        if self.importer_instance.source_file_alt:
            try:
                alt_xslt = etree.parse(self.importer_instance.source_file_alt)
                self.importer_instance.source_file_alt.seek(0)
                transform_child = etree.XSLT(alt_xslt)
            except (etree.XSLTParseError, etree.XMLSyntaxError, TypeError):
                return (0, 0,
                        _(u"The alt source file is not a valid XSLT file."))
        base_url = u"/".join(self.importer_instance.source.split(u'/')[:-1])
        base_url += u"/"
        for item in newdom.getroot():
            c_item = {child.tag: clean_field(child.text)
                      for child in item.getchildren() if child.text}
            # try to have more information on the linked page
            if transform_child and 'link' in c_item:
                # not an absolute address
                if not c_item['link'].startswith('http://') and \
                   not c_item['link'].startswith('https://'):
                    c_item['link'] = base_url + c_item['link']
                try:
                    child_page = urllib2.urlopen(c_item['link'])
                    assert child_page.getcode() == 200
                except (urllib2.URLError, AssertionError):
                    # don't stop the export for a bad link
                    items.append(c_item)
                    continue
                data = child_page.read()
                encoding = chardet.detect(data)
                data = data.decode(encoding['encoding'])
                child_page = BeautifulSoup(data).prettify()
                child_dom = etree.HTML(child_page, etree.HTMLParser())
                extra_keys = transform_child(child_dom).getroot()
                if len(extra_keys):
                    c_item.update({extra.tag: etree.tostring(extra)
                                   for extra in extra_keys[0].getchildren()})
            items.append(c_item)
        # change relative link to full link, simplify, unescape HTML entities
        html_unescape = HTMLParser.HTMLParser().unescape
        for item in items:
            for k in item:
                val = item[k]
                for r, replaced in RE_CLEANS:
                    val = re.sub(r, replaced % {'base_url': base_url}, val)
                item[k] = html_unescape(val)
        self.key_categories = self.importer_instance.get_key_category_dict()
        self.missing_cats = set()
        self.updated_item, self.new_item = 0, 0
        for item in items:
            self.add_dct_item(item)
        msg = ''
        if self.missing_cats:
            msg = _(
                u"Names \"%s\" doesn't match existing categories. "
                u"Modify the import to match theses names with categories.") %\
                (u'", "'.join(self.missing_cats))
        return (self.new_item, self.updated_item, msg)

    @classmethod
    def _internal_parse_date(cls, locale, year, month, day):
        try:
            year = datetime.date.today().year if not year else int(year)
        except ValueError:
            return
        month = month.encode('utf-8')
        if locale in MONTH_NAMES and month in MONTH_NAMES[locale]:
            month = MONTH_NAMES[locale].index(month) + 1
        else:
            try:
                month = int(month)
            except ValueError:
                return
        try:
            day = int(day)
        except ValueError:
            return
        try:
            return datetime.date(year, month, day)
        except ValueError:
            return

    def parse_date(self, date):
        dct = {}
        has_dates = False
        for locale in DATE_PARSINGS:
            if has_dates:
                break
            for r in DATE_PARSINGS[locale]:
                m = r.search(date)
                if not m:
                    continue
                values = m.groupdict()
                date = self._internal_parse_date(
                    locale, 'year1' in values and values['year1'],
                    values['month1'], values['day1'])
                if not date:
                    continue
                dct['start_date'] = date
                has_dates = True
                if 'day2' not in values:
                    break
                date = self._internal_parse_date(
                    locale, 'year2' in values and values['year2'],
                    values['month2'], values['day2'])
                if date:
                    dct['end_date'] = date
                break
        return dct

    def add_dct_item(self, item):
        if not self.importer_instance.default_localisation and \
                "point" not in item and not ("lat" in item and item['lat']):
            return
        cls = None
        dct = {
            'origin': "<a href='%s' target='_blank'>%s</a>" % (
                item.get('link') or '#', self.importer_instance.origin),
            'license': self.importer_instance.license,
            'name': item['name']}
        category = None
        if 'category' in item and item['category']:
            if item['category'] in self.key_categories:
                category = self.key_categories[item['category']]
            else:
                self.missing_cats.add(item['category'])
        cls = self.marker_cls
        if 'point' in item:
            x, y = item['point'].split(",")
            dct['point'] = 'SRID=4326;POINT(%s %s)' % (x, y)
        elif 'lat' in item and item['lat']:
            dct['point'] = 'SRID=4326;POINT(%s %s)' % (item['lon'],
                                                       item['lat'])
        else:
            dct['point'] = self.importer_instance.default_localisation
        dct['description'] = item.get('description', '')
        if 'date' in item:
            dct.update(self.parse_date(item['date']))
        key = item['key']
        it, updated, created = self.create_or_update_item(cls, dct, key,
                                                          category=category)
        if updated:
            self.updated_item += 1
        if created:
            self.new_item += 1


class XMLXsltManager(HtmlXsltManager):
    PARSER = 'XMLParser'