diff options
Diffstat (limited to 'chimere/utils.py')
| -rw-r--r-- | chimere/utils.py | 99 |
1 files changed, 45 insertions, 54 deletions
diff --git a/chimere/utils.py b/chimere/utils.py index bd09b9e..37c580f 100644 --- a/chimere/utils.py +++ b/chimere/utils.py @@ -25,16 +25,17 @@ import csv import collections import datetime import feedparser +import io import json import os import re -import StringIO import tempfile -import urllib2 +import urllib import unicodedata import zipfile from osgeo import ogr, osr +from osmapi import OsmApi from lxml import etree from django.conf import settings @@ -45,12 +46,9 @@ from django.shortcuts import render_to_response from django.utils.translation import ugettext_lazy as _ from chimere import get_version -from external_utils import OsmApi def unicode_normalize(string): - if type(string) == str: - string = unicode(string.decode('utf-8')) return ''.join( (c for c in unicodedata.normalize('NFD', string) if unicodedata.category(c) not in ('Mn', 'Sm', 'Sc'))) @@ -82,7 +80,7 @@ class ImportManager(object): key='', pk=None, category=None): from models import PropertyModel updated, created, item = False, False, None - import_key = unicode(import_key).replace(':', '^') + import_key = str(import_key).replace(':', '^') if not values.get('name'): values['name'] = self.default_name if not key: @@ -199,16 +197,16 @@ class ImportManager(object): url = source if extra_url: url += extra_url - remotehandle = urllib2.urlopen(url) - source = StringIO.StringIO(remotehandle.read()) + remotehandle = urllib.request.urlopen(url) + source = io.StringIO(remotehandle.read()) remotehandle.close() except ValueError: # assume it is a local file try: source = open(source) - except IOError, msg: + except IOError as msg: return (None, msg) - except (urllib2.URLError, AttributeError) as error: + except (urllib.error.URLError, AttributeError) as error: return (None, error.message) if self.importer_instance.zipped: try: @@ -256,7 +254,7 @@ class KMLManager(ImportManager): for idx, line in enumerate(splitted): if line.strip(): break - doc = StringIO.StringIO("\n".join(splitted[idx:])) + doc = io.StringIO("\n".join(splitted[idx:])) try: tree = etree.parse(doc) except: @@ -319,7 +317,7 @@ class KMLManager(ImportManager): def export(cls, queryset): dct = { 'name': settings.PROJECT_NAME, - 'description': unicode(datetime.date.today()), + 'description': str(datetime.date.today()), 'locations': queryset.all() } filename = unicode_normalize(settings.PROJECT_NAME + dct['description'] @@ -444,10 +442,10 @@ class ShapefileManager(ImportManager): for k in filtr: val = feat.get(k) try: - val = unicode(val) + val = str(val) except UnicodeDecodeError: try: - val = unicode( + val = str( val.decode(settings.CHIMERE_SHAPEFILE_ENCODING)) except: continue @@ -482,7 +480,7 @@ class ShapefileManager(ImportManager): @classmethod def export(cls, queryset): - date = unicode(datetime.date.today()) + date = str(datetime.date.today()) tmp = tempfile.NamedTemporaryFile(suffix='.shp', mode='w+b') tmp.close() @@ -496,7 +494,7 @@ class ShapefileManager(ImportManager): dr = ogr.GetDriverByName('ESRI Shapefile') ds = dr.CreateDataSource(tmp_name) if ds is None: - raise Exception(_(u'Could not create file!')) + raise Exception(_('Could not create file!')) ogr_type = OGRGeomType(geo_field.geom_type).num srs = osr.SpatialReference() srs.ImportFromEPSG(geo_field.srid) @@ -507,7 +505,7 @@ class ShapefileManager(ImportManager): field_defn = ogr.FieldDefn(str(field_name), ogr.OFTString) field_defn.SetWidth(255) if layer.CreateField(field_defn) != 0: - raise Exception(_(u'Failed to create field')) + raise Exception(_('Failed to create field')) feature_def = layer.GetLayerDefn() @@ -536,7 +534,7 @@ class ShapefileManager(ImportManager): # writing to a zip file filename = unicode_normalize(settings.PROJECT_NAME) + '-' + date - buff = StringIO.StringIO() + buff = io.StringIO() zip_file = zipfile.ZipFile(buff, 'w', zipfile.ZIP_DEFLATED) suffixes = ['shp', 'shx', 'prj', 'dbf'] for suffix in suffixes: @@ -632,7 +630,7 @@ class CSVManager(ImportManager): @classmethod def export(cls, queryset): - dct = {'description': unicode(datetime.date.today()), 'data': []} + dct = {'description': str(datetime.date.today()), 'data': []} # cls_name = queryset.model.__name__.lower() cols = list(cls.COLS) for pm in queryset.model.all_properties(): @@ -714,7 +712,7 @@ class GeoRSSManager(ImportManager): points = item['georss_line'].split(' ') reordered_points = [] # lat, lon -> x, y - for idx in xrange(len(points) / 2): + for idx in range(len(points) / 2): reordered_points.append("%s %s" % (points[idx * 2 + 1], points[idx * 2])) dct['route'] = 'SRID=4326;LINESTRING(%s)' % \ @@ -829,7 +827,7 @@ class JsonManager(ImportManager): dct[key] += filtr[k] cls = Marker pl_id = (dct.pop('id') if 'id' in dct else dct['name']) \ - + "-" + unicode(self.importer_instance.pk) + + "-" + str(self.importer_instance.pk) it, updated, created = self.create_or_update_item(cls, dct, pl_id) if updated: updated_item += 1 @@ -901,9 +899,9 @@ class OSMManager(ImportManager): dct = {'route': wkt, 'name': name, 'origin': self.importer_instance.origin - or u'OpenStreetMap.org', + or 'OpenStreetMap.org', 'license': self.importer_instance.license - or u'ODbL', + or 'ODbL', 'import_version': version} item, updated, created = self.create_or_update_item( Route, dct, node_id, version) @@ -932,9 +930,9 @@ class OSMManager(ImportManager): dct = {'point': point, 'name': name, 'origin': self.importer_instance.origin - or u'OpenStreetMap.org', + or 'OpenStreetMap.org', 'license': self.importer_instance.license - or u'ODbL', + or 'ODbL', 'import_version': version} item, updated, created = self.create_or_update_item( Marker, dct, node_id, version) @@ -1022,7 +1020,7 @@ class OSMManager(ImportManager): dct['version'] = item.import_version node = api.NodeUpdate(dct) updated = True - except OsmApi.ApiError, error: + except OsmApi.ApiError as error: if error.status == 404: dct.pop('id') dct.pop('version') @@ -1039,8 +1037,8 @@ class OSMManager(ImportManager): import chardet -import HTMLParser -from BeautifulSoup import BeautifulSoup +from html.parser import HTMLParser +from bs4 import BeautifulSoup RE_CLEANS = ((re.compile('(\n)*|^( )*(\n)*( )*|( )*(\n)*( )*$'), ''), @@ -1049,25 +1047,18 @@ RE_CLEANS = ((re.compile('(\n)*|^( )*(\n)*( )*|( )*(\n)*( )*$'), ''), '<a href="%(base_url)s\\1"'), ) -from calendar import TimeEncoding, month_name +from calendar import month_name, different_locale def get_month_name(month_no, locale): - with TimeEncoding(locale) as encoding: - s = month_name[month_no] - if encoding is not None: - s = s.decode(encoding) - return s + with different_locale(locale): + return month_name[month_no] MONTH_NAMES = {locale: [get_month_name(no_month, locale + '.UTF-8') - for no_month in xrange(1, 13)] for locale in ['fr_FR']} + for no_month in range(1, 13)] for locale in ['fr_FR']} -try: - UNI_MONTH_NAMES = {locale: [m.decode('utf-8') for m in MONTH_NAMES[locale]] - for locale in MONTH_NAMES} -except UnicodeEncodeError: - UNI_MONTH_NAMES = {locale: [m for m in MONTH_NAMES[locale]] - for locale in MONTH_NAMES} +UNI_MONTH_NAMES = {locale: [m for m in MONTH_NAMES[locale]] + for locale in MONTH_NAMES} DATE_PARSINGS = { 'fr_FR': [ @@ -1127,9 +1118,9 @@ class HtmlXsltManager(ImportManager): from models import Marker self.marker_cls = Marker try: - main_page = urllib2.urlopen(self.importer_instance.source) + main_page = urllib.request.urlopen(self.importer_instance.source) assert main_page.getcode() == 200 - except (urllib2.URLError, AssertionError): + except (urllib.error.URLError, AssertionError): return (0, 0, _(u"Source page is unreachable.")) data = main_page.read() encoding = chardet.detect(data) @@ -1159,7 +1150,7 @@ class HtmlXsltManager(ImportManager): except (etree.XSLTParseError, etree.XMLSyntaxError, TypeError): return (0, 0, _(u"The alt source file is not a valid XSLT file.")) - base_url = u"/".join(self.importer_instance.source.split(u'/')[:-1]) + base_url = u"/".join(self.importer_instance.source.split('/')[:-1]) base_url += u"/" for item in newdom.getroot(): c_item = {child.tag: clean_field(child.text) @@ -1171,9 +1162,9 @@ class HtmlXsltManager(ImportManager): not c_item['link'].startswith('https://'): c_item['link'] = base_url + c_item['link'] try: - child_page = urllib2.urlopen(c_item['link']) + child_page = urllib.request.urlopen(c_item['link']) assert child_page.getcode() == 200 - except (urllib2.URLError, AssertionError): + except (urllib.error.URLError, AssertionError): # don't stop the export for a bad link items.append(c_item) continue @@ -1188,7 +1179,7 @@ class HtmlXsltManager(ImportManager): for extra in extra_keys[0].getchildren()}) items.append(c_item) # change relative link to full link, simplify, unescape HTML entities - html_unescape = HTMLParser.HTMLParser().unescape + html_unescape = HTMLParser().unescape for item in items: for k in item: val = item[k] @@ -1205,7 +1196,7 @@ class HtmlXsltManager(ImportManager): msg = _( u"Names \"%s\" doesn't match existing categories. " u"Modify the import to match theses names with categories.") %\ - (u'", "'.join(self.missing_cats)) + ('", "'.join(self.missing_cats)) return (self.new_item, self.updated_item, msg) @classmethod @@ -1338,17 +1329,17 @@ class IcalManager(ImportManager): dct = default_dct.copy() dct['name'] = event.get('SUMMARY', '') if dct['name']: - dct['name'] = unicode(dct['name']) + dct['name'] = str(dct['name']) dct['description'] = event.get('DESCRIPTION', '') if dct['description']: - dct['description'] = unicode(dct['description']) + dct['description'] = str(dct['description']) loc = event.get('LOCATION', None) if loc: - dct['description'] += u"<br/>{}".format(unicode(loc)) + dct['description'] += u"<br/>{}".format(str(loc)) url = event.get('URL', None) if url: dct['description'] += u"<br/><a href='{}'>{}</a>".format( - unicode(url), unicode(_(u'Link'))) + str(url), unicode(_('Link'))) dct['start_date'] = event.get('DTSTART', None) if dct['start_date']: dct['start_date'] = event.decoded('DTSTART') @@ -1366,8 +1357,8 @@ class IcalManager(ImportManager): cls = Marker pl_id = event.get('UID', None) if not pl_id: - pl_id = dct['name'] + "-" + unicode(self.importer_instance.pk) - pl_id += "-" + unicode(self.importer_instance.pk) + pl_id = dct['name'] + "-" + str(self.importer_instance.pk) + pl_id += "-" + str(self.importer_instance.pk) it, updated, created = self.create_or_update_item(cls, dct, pl_id) if updated: updated_item += 1 |