From 8a8f25e156367807d174bbe858e8f66416306001 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Thu, 22 Sep 2016 23:06:12 +0200 Subject: Going to python3 and Django 1.8! --- chimere/utils.py | 99 ++++++++++++++++++++++++++------------------------------ 1 file changed, 45 insertions(+), 54 deletions(-) (limited to 'chimere/utils.py') diff --git a/chimere/utils.py b/chimere/utils.py index bd09b9e..37c580f 100644 --- a/chimere/utils.py +++ b/chimere/utils.py @@ -25,16 +25,17 @@ import csv import collections import datetime import feedparser +import io import json import os import re -import StringIO import tempfile -import urllib2 +import urllib import unicodedata import zipfile from osgeo import ogr, osr +from osmapi import OsmApi from lxml import etree from django.conf import settings @@ -45,12 +46,9 @@ from django.shortcuts import render_to_response from django.utils.translation import ugettext_lazy as _ from chimere import get_version -from external_utils import OsmApi def unicode_normalize(string): - if type(string) == str: - string = unicode(string.decode('utf-8')) return ''.join( (c for c in unicodedata.normalize('NFD', string) if unicodedata.category(c) not in ('Mn', 'Sm', 'Sc'))) @@ -82,7 +80,7 @@ class ImportManager(object): key='', pk=None, category=None): from models import PropertyModel updated, created, item = False, False, None - import_key = unicode(import_key).replace(':', '^') + import_key = str(import_key).replace(':', '^') if not values.get('name'): values['name'] = self.default_name if not key: @@ -199,16 +197,16 @@ class ImportManager(object): url = source if extra_url: url += extra_url - remotehandle = urllib2.urlopen(url) - source = StringIO.StringIO(remotehandle.read()) + remotehandle = urllib.request.urlopen(url) + source = io.StringIO(remotehandle.read()) remotehandle.close() except ValueError: # assume it is a local file try: source = open(source) - except IOError, msg: + except IOError as msg: return (None, msg) - except (urllib2.URLError, AttributeError) as error: + except (urllib.error.URLError, AttributeError) as error: return (None, error.message) if self.importer_instance.zipped: try: @@ -256,7 +254,7 @@ class KMLManager(ImportManager): for idx, line in enumerate(splitted): if line.strip(): break - doc = StringIO.StringIO("\n".join(splitted[idx:])) + doc = io.StringIO("\n".join(splitted[idx:])) try: tree = etree.parse(doc) except: @@ -319,7 +317,7 @@ class KMLManager(ImportManager): def export(cls, queryset): dct = { 'name': settings.PROJECT_NAME, - 'description': unicode(datetime.date.today()), + 'description': str(datetime.date.today()), 'locations': queryset.all() } filename = unicode_normalize(settings.PROJECT_NAME + dct['description'] @@ -444,10 +442,10 @@ class ShapefileManager(ImportManager): for k in filtr: val = feat.get(k) try: - val = unicode(val) + val = str(val) except UnicodeDecodeError: try: - val = unicode( + val = str( val.decode(settings.CHIMERE_SHAPEFILE_ENCODING)) except: continue @@ -482,7 +480,7 @@ class ShapefileManager(ImportManager): @classmethod def export(cls, queryset): - date = unicode(datetime.date.today()) + date = str(datetime.date.today()) tmp = tempfile.NamedTemporaryFile(suffix='.shp', mode='w+b') tmp.close() @@ -496,7 +494,7 @@ class ShapefileManager(ImportManager): dr = ogr.GetDriverByName('ESRI Shapefile') ds = dr.CreateDataSource(tmp_name) if ds is None: - raise Exception(_(u'Could not create file!')) + raise Exception(_('Could not create file!')) ogr_type = OGRGeomType(geo_field.geom_type).num srs = osr.SpatialReference() srs.ImportFromEPSG(geo_field.srid) @@ -507,7 +505,7 @@ class ShapefileManager(ImportManager): field_defn = ogr.FieldDefn(str(field_name), ogr.OFTString) field_defn.SetWidth(255) if layer.CreateField(field_defn) != 0: - raise Exception(_(u'Failed to create field')) + raise Exception(_('Failed to create field')) feature_def = layer.GetLayerDefn() @@ -536,7 +534,7 @@ class ShapefileManager(ImportManager): # writing to a zip file filename = unicode_normalize(settings.PROJECT_NAME) + '-' + date - buff = StringIO.StringIO() + buff = io.StringIO() zip_file = zipfile.ZipFile(buff, 'w', zipfile.ZIP_DEFLATED) suffixes = ['shp', 'shx', 'prj', 'dbf'] for suffix in suffixes: @@ -632,7 +630,7 @@ class CSVManager(ImportManager): @classmethod def export(cls, queryset): - dct = {'description': unicode(datetime.date.today()), 'data': []} + dct = {'description': str(datetime.date.today()), 'data': []} # cls_name = queryset.model.__name__.lower() cols = list(cls.COLS) for pm in queryset.model.all_properties(): @@ -714,7 +712,7 @@ class GeoRSSManager(ImportManager): points = item['georss_line'].split(' ') reordered_points = [] # lat, lon -> x, y - for idx in xrange(len(points) / 2): + for idx in range(len(points) / 2): reordered_points.append("%s %s" % (points[idx * 2 + 1], points[idx * 2])) dct['route'] = 'SRID=4326;LINESTRING(%s)' % \ @@ -829,7 +827,7 @@ class JsonManager(ImportManager): dct[key] += filtr[k] cls = Marker pl_id = (dct.pop('id') if 'id' in dct else dct['name']) \ - + "-" + unicode(self.importer_instance.pk) + + "-" + str(self.importer_instance.pk) it, updated, created = self.create_or_update_item(cls, dct, pl_id) if updated: updated_item += 1 @@ -901,9 +899,9 @@ class OSMManager(ImportManager): dct = {'route': wkt, 'name': name, 'origin': self.importer_instance.origin - or u'OpenStreetMap.org', + or 'OpenStreetMap.org', 'license': self.importer_instance.license - or u'ODbL', + or 'ODbL', 'import_version': version} item, updated, created = self.create_or_update_item( Route, dct, node_id, version) @@ -932,9 +930,9 @@ class OSMManager(ImportManager): dct = {'point': point, 'name': name, 'origin': self.importer_instance.origin - or u'OpenStreetMap.org', + or 'OpenStreetMap.org', 'license': self.importer_instance.license - or u'ODbL', + or 'ODbL', 'import_version': version} item, updated, created = self.create_or_update_item( Marker, dct, node_id, version) @@ -1022,7 +1020,7 @@ class OSMManager(ImportManager): dct['version'] = item.import_version node = api.NodeUpdate(dct) updated = True - except OsmApi.ApiError, error: + except OsmApi.ApiError as error: if error.status == 404: dct.pop('id') dct.pop('version') @@ -1039,8 +1037,8 @@ class OSMManager(ImportManager): import chardet -import HTMLParser -from BeautifulSoup import BeautifulSoup +from html.parser import HTMLParser +from bs4 import BeautifulSoup RE_CLEANS = ((re.compile('(\n)*|^( )*(\n)*( )*|( )*(\n)*( )*$'), ''), @@ -1049,25 +1047,18 @@ RE_CLEANS = ((re.compile('(\n)*|^( )*(\n)*( )*|( )*(\n)*( )*$'), ''), '{}".format(unicode(loc)) + dct['description'] += u"
{}".format(str(loc)) url = event.get('URL', None) if url: dct['description'] += u"
{}".format( - unicode(url), unicode(_(u'Link'))) + str(url), unicode(_('Link'))) dct['start_date'] = event.get('DTSTART', None) if dct['start_date']: dct['start_date'] = event.decoded('DTSTART') @@ -1366,8 +1357,8 @@ class IcalManager(ImportManager): cls = Marker pl_id = event.get('UID', None) if not pl_id: - pl_id = dct['name'] + "-" + unicode(self.importer_instance.pk) - pl_id += "-" + unicode(self.importer_instance.pk) + pl_id = dct['name'] + "-" + str(self.importer_instance.pk) + pl_id += "-" + str(self.importer_instance.pk) it, updated, created = self.create_or_update_item(cls, dct, pl_id) if updated: updated_item += 1 -- cgit v1.2.3