Going to python3 and Django 1.8!

1 files changed, 45 insertions, 54 deletions

diff --git a/chimere/utils.py b/chimere/utils.py
index bd09b9e..37c580f 100644
--- a/chimere/utils.py
+++ b/chimere/utils.py
@@ -25,16 +25,17 @@ import csv
 import collections
 import datetime
 import feedparser
+import io
 import json
 import os
 import re
-import StringIO
 import tempfile
-import urllib2
+import urllib
 import unicodedata
 import zipfile
 
 from osgeo import ogr, osr
+from osmapi import OsmApi
 from lxml import etree
 
 from django.conf import settings
@@ -45,12 +46,9 @@ from django.shortcuts import render_to_response
 from django.utils.translation import ugettext_lazy as _
 
 from chimere import get_version
-from external_utils import OsmApi
 
 
 def unicode_normalize(string):
-    if type(string) == str:
-        string = unicode(string.decode('utf-8'))
     return ''.join(
         (c for c in unicodedata.normalize('NFD', string)
          if unicodedata.category(c) not in ('Mn', 'Sm', 'Sc')))
@@ -82,7 +80,7 @@ class ImportManager(object):
                               key='', pk=None, category=None):
         from models import PropertyModel
         updated, created, item = False, False, None
-        import_key = unicode(import_key).replace(':', '^')
+        import_key = str(import_key).replace(':', '^')
         if not values.get('name'):
             values['name'] = self.default_name
         if not key:
@@ -199,16 +197,16 @@ class ImportManager(object):
                 url = source
                 if extra_url:
                     url += extra_url
-                remotehandle = urllib2.urlopen(url)
-                source = StringIO.StringIO(remotehandle.read())
+                remotehandle = urllib.request.urlopen(url)
+                source = io.StringIO(remotehandle.read())
                 remotehandle.close()
             except ValueError:
                 # assume it is a local file
                 try:
                     source = open(source)
-                except IOError, msg:
+                except IOError as msg:
                     return (None, msg)
-            except (urllib2.URLError, AttributeError) as error:
+            except (urllib.error.URLError, AttributeError) as error:
                 return (None, error.message)
         if self.importer_instance.zipped:
             try:
@@ -256,7 +254,7 @@ class KMLManager(ImportManager):
             for idx, line in enumerate(splitted):
                 if line.strip():
                     break
-            doc = StringIO.StringIO("\n".join(splitted[idx:]))
+            doc = io.StringIO("\n".join(splitted[idx:]))
         try:
             tree = etree.parse(doc)
         except:
@@ -319,7 +317,7 @@ class KMLManager(ImportManager):
     def export(cls, queryset):
         dct = {
             'name': settings.PROJECT_NAME,
-            'description': unicode(datetime.date.today()),
+            'description': str(datetime.date.today()),
             'locations': queryset.all()
         }
         filename = unicode_normalize(settings.PROJECT_NAME + dct['description']
@@ -444,10 +442,10 @@ class ShapefileManager(ImportManager):
             for k in filtr:
                 val = feat.get(k)
                 try:
-                    val = unicode(val)
+                    val = str(val)
                 except UnicodeDecodeError:
                     try:
-                        val = unicode(
+                        val = str(
                             val.decode(settings.CHIMERE_SHAPEFILE_ENCODING))
                     except:
                         continue
@@ -482,7 +480,7 @@ class ShapefileManager(ImportManager):
 
     @classmethod
     def export(cls, queryset):
-        date = unicode(datetime.date.today())
+        date = str(datetime.date.today())
 
         tmp = tempfile.NamedTemporaryFile(suffix='.shp', mode='w+b')
         tmp.close()
@@ -496,7 +494,7 @@ class ShapefileManager(ImportManager):
         dr = ogr.GetDriverByName('ESRI Shapefile')
         ds = dr.CreateDataSource(tmp_name)
         if ds is None:
-            raise Exception(_(u'Could not create file!'))
+            raise Exception(_('Could not create file!'))
         ogr_type = OGRGeomType(geo_field.geom_type).num
         srs = osr.SpatialReference()
         srs.ImportFromEPSG(geo_field.srid)
@@ -507,7 +505,7 @@ class ShapefileManager(ImportManager):
             field_defn = ogr.FieldDefn(str(field_name), ogr.OFTString)
             field_defn.SetWidth(255)
             if layer.CreateField(field_defn) != 0:
-                raise Exception(_(u'Failed to create field'))
+                raise Exception(_('Failed to create field'))
 
         feature_def = layer.GetLayerDefn()
 
@@ -536,7 +534,7 @@ class ShapefileManager(ImportManager):
 
         # writing to a zip file
         filename = unicode_normalize(settings.PROJECT_NAME) + '-' + date
-        buff = StringIO.StringIO()
+        buff = io.StringIO()
         zip_file = zipfile.ZipFile(buff, 'w', zipfile.ZIP_DEFLATED)
         suffixes = ['shp', 'shx', 'prj', 'dbf']
         for suffix in suffixes:
@@ -632,7 +630,7 @@ class CSVManager(ImportManager):
 
     @classmethod
     def export(cls, queryset):
-        dct = {'description': unicode(datetime.date.today()), 'data': []}
+        dct = {'description': str(datetime.date.today()), 'data': []}
         # cls_name = queryset.model.__name__.lower()
         cols = list(cls.COLS)
         for pm in queryset.model.all_properties():
@@ -714,7 +712,7 @@ class GeoRSSManager(ImportManager):
                 points = item['georss_line'].split(' ')
                 reordered_points = []
                 # lat, lon -> x, y
-                for idx in xrange(len(points) / 2):
+                for idx in range(len(points) / 2):
                     reordered_points.append("%s %s" % (points[idx * 2 + 1],
                                                        points[idx * 2]))
                 dct['route'] = 'SRID=4326;LINESTRING(%s)' % \
@@ -829,7 +827,7 @@ class JsonManager(ImportManager):
                             dct[key] += filtr[k]
             cls = Marker
             pl_id = (dct.pop('id') if 'id' in dct else dct['name']) \
-                + "-" + unicode(self.importer_instance.pk)
+                + "-" + str(self.importer_instance.pk)
             it, updated, created = self.create_or_update_item(cls, dct, pl_id)
             if updated:
                 updated_item += 1
@@ -901,9 +899,9 @@ class OSMManager(ImportManager):
             dct = {'route': wkt,
                    'name': name,
                    'origin': self.importer_instance.origin
-                   or u'OpenStreetMap.org',
+                   or 'OpenStreetMap.org',
                    'license': self.importer_instance.license
-                   or u'ODbL',
+                   or 'ODbL',
                    'import_version': version}
             item, updated, created = self.create_or_update_item(
                 Route, dct, node_id, version)
@@ -932,9 +930,9 @@ class OSMManager(ImportManager):
             dct = {'point': point,
                    'name': name,
                    'origin': self.importer_instance.origin
-                   or u'OpenStreetMap.org',
+                   or 'OpenStreetMap.org',
                    'license': self.importer_instance.license
-                   or u'ODbL',
+                   or 'ODbL',
                    'import_version': version}
             item, updated, created = self.create_or_update_item(
                 Marker, dct, node_id, version)
@@ -1022,7 +1020,7 @@ class OSMManager(ImportManager):
                     dct['version'] = item.import_version
                     node = api.NodeUpdate(dct)
                     updated = True
-                except OsmApi.ApiError, error:
+                except OsmApi.ApiError as error:
                     if error.status == 404:
                         dct.pop('id')
                         dct.pop('version')
@@ -1039,8 +1037,8 @@ class OSMManager(ImportManager):
 
 
 import chardet
-import HTMLParser
-from BeautifulSoup import BeautifulSoup
+from html.parser import HTMLParser
+from bs4 import BeautifulSoup
 
 
 RE_CLEANS = ((re.compile('(\n)*|^( )*(\n)*( )*|( )*(\n)*( )*$'), ''),
@@ -1049,25 +1047,18 @@ RE_CLEANS = ((re.compile('(\n)*|^( )*(\n)*( )*|( )*(\n)*( )*$'), ''),
               '<a href="%(base_url)s\\1"'),
              )
 
-from calendar import TimeEncoding, month_name
+from calendar import month_name, different_locale
 
 
 def get_month_name(month_no, locale):
-    with TimeEncoding(locale) as encoding:
-        s = month_name[month_no]
-        if encoding is not None:
-            s = s.decode(encoding)
-        return s
+    with different_locale(locale):
+        return month_name[month_no]
 
 MONTH_NAMES = {locale: [get_month_name(no_month, locale + '.UTF-8')
-               for no_month in xrange(1, 13)] for locale in ['fr_FR']}
+               for no_month in range(1, 13)] for locale in ['fr_FR']}
 
-try:
-    UNI_MONTH_NAMES = {locale: [m.decode('utf-8') for m in MONTH_NAMES[locale]]
-                       for locale in MONTH_NAMES}
-except UnicodeEncodeError:
-    UNI_MONTH_NAMES = {locale: [m for m in MONTH_NAMES[locale]]
-                       for locale in MONTH_NAMES}
+UNI_MONTH_NAMES = {locale: [m for m in MONTH_NAMES[locale]]
+                   for locale in MONTH_NAMES}
 
 DATE_PARSINGS = {
     'fr_FR': [
@@ -1127,9 +1118,9 @@ class HtmlXsltManager(ImportManager):
         from models import Marker
         self.marker_cls = Marker
         try:
-            main_page = urllib2.urlopen(self.importer_instance.source)
+            main_page = urllib.request.urlopen(self.importer_instance.source)
             assert main_page.getcode() == 200
-        except (urllib2.URLError, AssertionError):
+        except (urllib.error.URLError, AssertionError):
             return (0, 0, _(u"Source page is unreachable."))
         data = main_page.read()
         encoding = chardet.detect(data)
@@ -1159,7 +1150,7 @@ class HtmlXsltManager(ImportManager):
             except (etree.XSLTParseError, etree.XMLSyntaxError, TypeError):
                 return (0, 0,
                         _(u"The alt source file is not a valid XSLT file."))
-        base_url = u"/".join(self.importer_instance.source.split(u'/')[:-1])
+        base_url = u"/".join(self.importer_instance.source.split('/')[:-1])
         base_url += u"/"
         for item in newdom.getroot():
             c_item = {child.tag: clean_field(child.text)
@@ -1171,9 +1162,9 @@ class HtmlXsltManager(ImportManager):
                    not c_item['link'].startswith('https://'):
                     c_item['link'] = base_url + c_item['link']
                 try:
-                    child_page = urllib2.urlopen(c_item['link'])
+                    child_page = urllib.request.urlopen(c_item['link'])
                     assert child_page.getcode() == 200
-                except (urllib2.URLError, AssertionError):
+                except (urllib.error.URLError, AssertionError):
                     # don't stop the export for a bad link
                     items.append(c_item)
                     continue
@@ -1188,7 +1179,7 @@ class HtmlXsltManager(ImportManager):
                                    for extra in extra_keys[0].getchildren()})
             items.append(c_item)
         # change relative link to full link, simplify, unescape HTML entities
-        html_unescape = HTMLParser.HTMLParser().unescape
+        html_unescape = HTMLParser().unescape
         for item in items:
             for k in item:
                 val = item[k]
@@ -1205,7 +1196,7 @@ class HtmlXsltManager(ImportManager):
             msg = _(
                 u"Names \"%s\" doesn't match existing categories. "
                 u"Modify the import to match theses names with categories.") %\
-                (u'", "'.join(self.missing_cats))
+                ('", "'.join(self.missing_cats))
         return (self.new_item, self.updated_item, msg)
 
     @classmethod
@@ -1338,17 +1329,17 @@ class IcalManager(ImportManager):
             dct = default_dct.copy()
             dct['name'] = event.get('SUMMARY', '')
             if dct['name']:
-                dct['name'] = unicode(dct['name'])
+                dct['name'] = str(dct['name'])
             dct['description'] = event.get('DESCRIPTION', '')
             if dct['description']:
-                dct['description'] = unicode(dct['description'])
+                dct['description'] = str(dct['description'])
             loc = event.get('LOCATION', None)
             if loc:
-                dct['description'] += u"<br/>{}".format(unicode(loc))
+                dct['description'] += u"<br/>{}".format(str(loc))
             url = event.get('URL', None)
             if url:
                 dct['description'] += u"<br/><a href='{}'>{}</a>".format(
-                    unicode(url), unicode(_(u'Link')))
+                    str(url), unicode(_('Link')))
             dct['start_date'] = event.get('DTSTART', None)
             if dct['start_date']:
                 dct['start_date'] = event.decoded('DTSTART')
@@ -1366,8 +1357,8 @@ class IcalManager(ImportManager):
             cls = Marker
             pl_id = event.get('UID', None)
             if not pl_id:
-                pl_id = dct['name'] + "-" + unicode(self.importer_instance.pk)
-            pl_id += "-" + unicode(self.importer_instance.pk)
+                pl_id = dct['name'] + "-" + str(self.importer_instance.pk)
+            pl_id += "-" + str(self.importer_instance.pk)
             it, updated, created = self.create_or_update_item(cls, dct, pl_id)
             if updated:
                 updated_item += 1