The ODT file to use as a template (must be readable).
.. cmdoption:: -v
Be verbose (enables logging)
The full help message is::
Usage: xhtml2odt.py [options] -i input -o output -t template.odt
Options:
-h, --help show this help message and exit
-i FILE, --input=FILE
Read the html from this file
-o FILE, --output=FILE
Location of the output ODT file
-t FILE, --template=FILE
Location of the template ODT file
-u URL, --url=URL Use this URL for relative links
-v, --verbose Show what's going on
--html-id=ID Only export from the element with this ID
--replace=KEYWORD Keyword to replace in the ODT template (default is
ODT-INSERT)
--cut-start=KEYWORD Keyword to start cutting text from the ODT template
(default is ODT-CUT-START)
--cut-stop=KEYWORD Keyword to stop cutting text from the ODT template
(default is ODT-CUT-STOP)
--top-header-level=LEVEL
Level of highest header in the HTML (default is 1)
--img-default-width=WIDTH
Default image width (default is 8cm)
--img-default-height=HEIGHT
Default image height (default is 6cm)
--dpi=DPI Screen resolution in Dots Per Inch (default is 96)
--no-network Do not download remote images
License
-------
GNU LGPL v2.1 or later: http://www.gnu.org/licenses/lgpl-2.1.html
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Library General Public License for more details.
Code
----
"""
import tempfile
import shutil
import re
import os
import sys
import zipfile
import urllib2
import urlparse
import hashlib
import mimetypes
from StringIO import StringIO
from optparse import OptionParser
try:
import tidy
except:
from tidylib import tidy_document as tidy
from lxml import etree
from PIL import Image
#pylint#: disable-msg=C0301,C0111
INSTALL_PATH = os.path.dirname(__file__)
INCH_TO_CM = 2.54
CHARSET = "utf-8"
__version__ = 0.1
class ODTExportError(Exception):
"""Base exception for ODT conversion errors"""
pass
class HTMLFile(object):
"""
This class contains the HTML document to convert to ODT. The HTML code
will be run through Tidy to ensure that is is valid and well-formed
XHTML.
:ivar options: An OptionParser-result object containing the options for
processing.
:type options: OptionsParser-result object
:ivar html: The HTML code.
:type html: ``str``
"""
def __init__(self, options):
self.options = options
self.html = ""
def read(self):
"""
Read the HTML file from :attr:`options`.input, run it through Tidy, and
filter using the selected ID (if applicable).
"""
in_file = open(self.options.input)
self.html = in_file.read()
in_file.close()
self.cleanup()
if self.options.htmlid:
self.select_id()
def cleanup(self):
"""
Run the HTML code from the :attr:`html` instance variable through Tidy.
"""
tidy_options = dict(output_xhtml=1, add_xml_decl=1, indent=1,
tidy_mark=0, #input_encoding=str(self.charset),
output_encoding='utf8', doctype='auto',
wrap=0, char_encoding='utf8')
self.html = str(tidy.parseString(self.html, **tidy_options))
if not self.html:
raise ODTExportError(
"Tidy could not clean up the document, aborting.")
# Replace nbsp with entity
# http://www.mail-archive.com/analog-help@lists.meer.net/msg03670.html
self.html = self.html.replace(" ", " ")
# Tidy creates newlines after (by indenting)
self.html = re.sub(']*)>\n', '', self.html)
def select_id(self):
"""
Replace the HTML content by an element in the content. The element
is selected by its HTML ID.
"""
try:
html_tree = etree.fromstring(self.html)
except etree.XMLSyntaxError:
if self.options.verbose:
raise
else:
raise ODTExportError("The XHTML is still not valid after "
"Tidy's work, I can't convert it.")
selected = html_tree.xpath("//*[@id='%s']" % self.options.htmlid)
if selected:
self.html = etree.tostring(selected[0], method="html")
else:
print >> sys.stderr, "Can't find the selected HTML id: %s, " \
% self.options.htmlid \
+"converting everything."
class ODTFile(object):
"""Handles the conversion and production of an ODT file"""
def __init__(self, options):
self.options = options
self.xml = {
"content": "",
"styles": "",
}
self.tmpdir = tempfile.mkdtemp(prefix="xhtml2odt-")
self.zfile = None
self._added_images = []
def open(self):
"""
Uncompress the template ODT file, and read the content.xml and
styles.xml files into memory.
"""
self.zfile = zipfile.ZipFile(self.options.template, "r")
for name in self.zfile.namelist():
fname = os.path.join(self.tmpdir, name)
if not os.path.exists(os.path.dirname(fname)):
os.makedirs(os.path.dirname(fname))
if name[-1] == "/":
if not os.path.exists(fname):
os.mkdir(fname)
continue
fname_h = open(fname, "w")
fname_h.write(self.zfile.read(name))
fname_h.close()
for xmlfile in self.xml:
self.xml[xmlfile] = self.zfile.read("%s.xml" % xmlfile)
def import_xhtml(self, xhtml):
"""
Main function to run the conversion process:
* XHTML import
* conversion to ODT XML
* insertion into the ODT template
* adding of the missing styles
The next logical step is to use the :meth:`save` method.
:param xhtml: the XHTML content to import
:type xhtml: str
"""
odt = self.xhtml_to_odt(xhtml)
self.insert_content(odt)
self.add_styles()
def xhtml_to_odt(self, xhtml):
"""
Converts the XHTML content into ODT.
:param xhtml: the XHTML content to import
:type xhtml: str
:returns: the ODT XML from the conversion
:rtype: str
"""
xsl_dir = os.path.join(INSTALL_PATH, 'xsl')
xslt_doc = etree.parse(os.path.join(xsl_dir, "xhtml2odt.xsl"))
transform = etree.XSLT(xslt_doc)
xhtml = self.handle_images(xhtml)
xhtml = self.handle_links(xhtml)
try:
xhtml = etree.fromstring(xhtml) # must be valid xml at this point
except etree.XMLSyntaxError:
if self.options.verbose:
raise
else:
raise ODTExportError("The XHTML is still not valid after "
"Tidy's work, I can't convert it.")
params = {
"url": "/",
"heading_minus_level": str(self.options.top_header_level - 1),
}
if self.options.verbose:
params["debug"] = "1"
if self.options.img_width:
if hasattr(etree.XSLT, "strparam"):
params["img_default_width"] = etree.XSLT.strparam(
self.options.img_width)
else: # lxml < 2.2
params["img_default_width"] = "'%s'" % self.options.img_width
if self.options.img_height:
if hasattr(etree.XSLT, "strparam"):
params["img_default_height"] = etree.XSLT.strparam(
self.options.img_height)
else: # lxml < 2.2
params["img_default_height"] = "'%s'" % self.options.img_height
odt = transform(xhtml, **params)
# DEBUG
#print str(odt)
return str(odt).replace('','')
def handle_images(self, xhtml):
"""
Handling of image tags in the XHTML. Local and remote images are
handled differently: see the :meth:`handle_local_img` and
:meth:`handle_remote_img` methods for details.
:param xhtml: the XHTML content to import
:type xhtml: str
:returns: XHTML with normalized ``img`` tags
:rtype: str
"""
# Handle local images
xhtml = re.sub('![]()
]*src="([^"]+)"[^>]*>',
self.handle_local_img, xhtml)
# Handle remote images
if self.options.with_network:
xhtml = re.sub('![]()
]*src="(https?://[^"]+)"[^>]*>',
self.handle_remote_img, xhtml)
#print xhtml
return xhtml
def handle_local_img(self, img_mo):
"""
Handling of local images. This method should be called as a callback on
each ``img`` tag.
Find the real path of the image file and use the :meth:`handle_img`
method to flag it for inclusion in the ODT file.
This implementation downloads the files that come from the same domain
as the XHTML document cames from, but server-based export plugins can
just retrieve it from the local disk, using either the
``DOCUMENT_ROOT`` or any appropriate method (depending on the web
application you're writing an export plugin for).
:param img_mo: the match object from the `re.sub` callback
"""
log("handling local image: %s" % img_mo.group(1), self.options.verbose)
src = img_mo.group(1)
if src.count("://") and not src.startswith("file://"):
# This is an absolute link, don't touch it
return img_mo.group()
if src.startswith("file://"):
filename = src[7:]
elif src.startswith("/"):
filename = src
else: # relative link
filename = os.path.join(os.path.dirname(self.options.input), src)
if os.path.exists(filename):
return self.handle_img(img_mo.group(), src, filename)
if src.startswith("file://") or not self.options.url:
# There's nothing we can do here
return img_mo.group()
newsrc = urlparse.urljoin(self.options.url, os.path.normpath(src))
if not self.options.with_network:
# Don't download it, just update the URL
return img_mo.group().replace(src, newsrc)
try:
tmpfile = self.download_img(newsrc)
except (urllib2.HTTPError, urllib2.URLError):
log("Failed getting %s" % newsrc, self.options.verbose)
return img_mo.group()
ret = self.handle_img(img_mo.group(), src, tmpfile)
os.remove(tmpfile)
return ret
def handle_remote_img(self, img_mo):
"""
Downloads remote images to a temporary file and flags them for
inclusion using the :meth:`handle_img` method.
:param img_mo: the match object from the `re.sub` callback
"""
log('handling remote image: %s' % img_mo.group(), self.options.verbose)
src = img_mo.group(1)
try:
tmpfile = self.download_img(src)
except (urllib2.HTTPError, urllib2.URLError):
return img_mo.group()
ret = self.handle_img(img_mo.group(), src, tmpfile)
os.remove(tmpfile)
return ret
def download_img(self, src):
"""
Downloads the given image to a temporary location.
:param src: the URL to download
:type src: str
"""
log('Downloading image: %s' % src, self.options.verbose)
# TODO: proxy support
remoteimg = urllib2.urlopen(src)
tmpimg_fd, tmpfile = tempfile.mkstemp()
tmpimg = os.fdopen(tmpimg_fd, 'w')
tmpimg.write(remoteimg.read())
tmpimg.close()
remoteimg.close()
return tmpfile
def handle_img(self, full_tag, src, filename):
"""
Imports an image into the ODT file.
:param full_tag: the full ``img`` tag in the original XHTML document
:type full_tag: str
:param src: the ``src`` attribute of the ``img`` tag
:type src: str
:param filename: the path to the image file on the local disk
:type filename: str
"""
log('Importing image: %s' % filename, self.options.verbose)
if not os.path.exists(filename):
raise ODTExportError('Image "%s" is not readable or does not exist'
% filename)
# TODO: generate a filename (with tempfile.mkstemp) to avoid weird
# filenames. Maybe use img.format for the extension
if not os.path.exists(os.path.join(self.tmpdir, "Pictures")):
os.mkdir(os.path.join(self.tmpdir, "Pictures"))
newname = ( hashlib.md5(filename).hexdigest()
+ os.path.splitext(filename)[1] )
shutil.copy(filename, os.path.join(self.tmpdir, "Pictures", newname))
self._added_images.append(os.path.join("Pictures", newname))
full_tag = full_tag.replace('src="%s"' % src,
'src="Pictures/%s"' % newname)
try:
img = Image.open(filename)
except IOError:
log('Failed to identify image: %s' % filename,
self.options.verbose)
else:
width, height = img.size
log('Detected size: %spx x %spx' % (width, height),
self.options.verbose)
width_mo = re.search('width="([0-9]+)(?:px)?"', full_tag)
height_mo = re.search('height="([0-9]+)(?:px)?"', full_tag)
if width_mo and height_mo:
log('Forced size: %spx x %spx.' % (width_mo.group(),
height_mo.group()), self.options.verbose)
width = float(width_mo.group(1)) / self.options.img_dpi \
* INCH_TO_CM
height = float(height_mo.group(1)) / self.options.img_dpi \
* INCH_TO_CM
full_tag = full_tag.replace(width_mo.group(), "")\
.replace(height_mo.group(), "")
elif width_mo and not height_mo:
newwidth = float(width_mo.group(1)) / \
float(self.options.img_dpi) * INCH_TO_CM
height = height * newwidth / width
width = newwidth
log('Forced width: %spx. Size will be: %scm x %scm' %
(width_mo.group(1), width, height), self.options.verbose)
full_tag = full_tag.replace(width_mo.group(), "")
elif not width_mo and height_mo:
newheight = float(height_mo.group(1)) / \
float(self.options.img_dpi) * INCH_TO_CM
width = width * newheight / height
height = newheight
log('Forced height: %spx. Size will be: %scm x %scm' %
(height_mo.group(1), height, width), self.options.verbose)
full_tag = full_tag.replace(height_mo.group(), "")
else:
width = width / float(self.options.img_dpi) * INCH_TO_CM
height = height / float(self.options.img_dpi) * INCH_TO_CM
log('Size converted to: %scm x %scm' % (height, width),
self.options.verbose)
full_tag = full_tag.replace(']*href="([^"]+)"',
self.handle_relative_links, xhtml)
return xhtml
def handle_relative_links(self, link_mo):
"""
Do the actual conversion of links from relative to absolute. This
method is used as a callback by the :meth:`handle_links` method.
"""
href = link_mo.group(1)
if href.startswith("file://") or not self.options.url:
# There's nothing we can do here
return link_mo.group()
if href.count("://"):
# This is an absolute link, don't touch it
return link_mo.group()
log("handling relative link: %s" % href, self.options.verbose)
newhref = urlparse.urljoin(self.options.url, os.path.normpath(href))
return link_mo.group().replace(href, newhref)
def insert_content(self, content):
"""
Insert ODT XML content into the ``content.xml`` file, replacing the
keywords if needed.
:param content: ODT XML content to insert
:type content: str
"""
if self.options.replace_keyword and \
self.xml["content"].count(self.options.replace_keyword) > 0:
self.xml["content"] = re.sub(
"]*>" +
re.escape(self.options.replace_keyword)
+"", content, self.xml["content"])
else:
self.xml["content"] = self.xml["content"].replace(
'',
content + '')
# Cut unwanted text
if self.options.cut_start \
and self.xml["content"].count(self.options.cut_start) > 0 \
and self.options.cut_stop \
and self.xml["content"].count(self.options.cut_stop) > 0:
self.xml["content"] = re.sub(
re.escape(self.options.cut_start)
+ ".*" +
re.escape(self.options.cut_stop),
"", self.xml["content"])
def add_styles(self):
"""
Scans the ODT XML for used styles that would not be already included in
the ODT template, and adds those missing styles.
"""
xsl_dir = os.path.join(INSTALL_PATH, 'xsl')
xslt_doc = etree.parse(os.path.join(xsl_dir, "styles.xsl"))
transform = etree.XSLT(xslt_doc)
contentxml = etree.fromstring(self.xml["content"])
stylesxml = etree.fromstring(self.xml["styles"])
params = {}
if self.options.verbose:
params["debug"] = "1"
self.xml["content"] = str(transform(contentxml, **params))
self.xml["styles"] = str(transform(stylesxml, **params))
def update_manifest(self):
manifest_path = os.path.join(self.tmpdir, "META-INF", "manifest.xml")
if not os.path.exists(manifest_path):
return
manifest = etree.parse(manifest_path)
manifest_root = manifest.getroot()
manifest_ns = "urn:oasis:names:tc:opendocument:xmlns:manifest:1.0"
for img in self._added_images:
mime_type = mimetypes.guess_type(img, strict=False)[0]
if mime_type is None:
continue
img_el = etree.SubElement(
manifest_root,
"{%s}file-entry" % manifest_ns,
{"{%s}media-type" % manifest_ns: mime_type,
"{%s}full-path" % manifest_ns: img,
})
manifest.write(manifest_path)
def compile(self):
"""
Writes the in-memory ODT XML content and styles to the disk
"""
# Store the new content
for xmlfile in self.xml:
xmlf = open(os.path.join(self.tmpdir, "%s.xml" % xmlfile), "w")
xmlf.write(self.xml[xmlfile])
xmlf.close()
self.update_manifest()
def _build_zip(self, document):
"""
Zips the working directory into a :class:`zipfile.ZipFile` object
:param document: where the :class:`ZipFile` will be stored
:type document: str or file-like object
"""
newzf = zipfile.ZipFile(document, "w", zipfile.ZIP_DEFLATED)
for root, dirs, files in os.walk(self.tmpdir):
for cur_file in files:
realpath = os.path.join(root, cur_file)
to_skip = len(self.tmpdir) + 1
internalpath = os.path.join(root[to_skip:], cur_file)
newzf.write(realpath, internalpath)
newzf.close()
def save(self, output=None):
"""
General method to save the in-memory content to an ODT file on the disk.
If :attr:`output` is ``None``, the document is returned.
:param output: where the document should be saved, see the :option:`-o`
option.
:type output: str or file-like object or ``None``
:returns: if output is None: the ODT document ; or else ``None``.
"""
self.compile()
if output:
document = output
else:
document = StringIO()
self._build_zip(document)
shutil.rmtree(self.tmpdir)
if not output:
return document.getvalue()
def log(msg, verbose=False):
"""
Simple method to log if we're in verbose mode (with the :option:`-v`
option).
"""
if verbose:
sys.stderr.write(msg+"\n")
def get_options():
"""
Parses the command-line options.
"""
usage = "usage: %prog [options] -i input -o output -t template.odt"
parser = OptionParser(usage=usage)
parser.add_option("--version", dest="version", action="store_true",
help="Show the version and exit")
parser.add_option("-i", "--input", dest="input", metavar="FILE",
help="Read the html from this file")
parser.add_option("-o", "--output", dest="output", metavar="FILE",
help="Location of the output ODT file")
parser.add_option("-t", "--template", dest="template", metavar="FILE",
help="Location of the template ODT file")
parser.add_option("-u", "--url", dest="url",
help="Use this URL for relative links")
parser.add_option("-v", "--verbose", dest="verbose",
action="store_true", default=False,
help="Show what's going on")
parser.add_option("--html-id", dest="htmlid", metavar="ID",
help="Only export from the element with this ID")
parser.add_option("--replace", dest="replace_keyword",
default="ODT-INSERT", metavar="KEYWORD",
help="Keyword to replace in the ODT template "
"(default is %default)")
parser.add_option("--cut-start", dest="cut_start",
default="ODT-CUT-START", metavar="KEYWORD",
help="Keyword to start cutting text from the ODT "
"template (default is %default)")
parser.add_option("--cut-stop", dest="cut_stop",
default="ODT-CUT-STOP", metavar="KEYWORD",
help="Keyword to stop cutting text from the ODT "
"template (default is %default)")
parser.add_option("--top-header-level", dest="top_header_level",
type="int", default="1", metavar="LEVEL",
help="Level of highest header in the HTML "
"(default is %default)")
parser.add_option("--img-default-width", dest="img_width",
metavar="WIDTH", default="8cm",
help="Default image width (default is %default)")
parser.add_option("--img-default-height", dest="img_height",
metavar="HEIGHT", default="6cm",
help="Default image height (default is %default)")
parser.add_option("--dpi", dest="img_dpi", type="int",
default=96, metavar="DPI", help="Screen resolution "
"in Dots Per Inch (default is %default)")
parser.add_option("--no-network", dest="with_network",
action="store_false", default=True,
help="Do not download remote images")
options, args = parser.parse_args()
if options.version:
print "xhtml2odt %s" % __version__
sys.exit(0)
if len(args) > 0:
parser.error("illegal arguments: %s"% ", ".join(args))
if not options.input:
parser.error("No input provided")
if not options.output:
parser.error("No output provided")
if not options.template:
default_template = os.path.join(INSTALL_PATH, "template.odt")
if os.path.exists(default_template):
options.template = default_template
else:
parser.error("No ODT template provided")
if not os.path.exists(options.input):
parser.error("Can't find input file: %s" % options.input)
if not os.path.exists(options.template):
parser.error("Can't find template file: %s" % options.template)
return options
def main():
"""
Main function, called when the script is invoked on the command line.
"""
options = get_options()
try:
htmlfile = HTMLFile(options)
htmlfile.read()
odtfile = ODTFile(options)
odtfile.open()
odtfile.import_xhtml(htmlfile.html)
odtfile.save(options.output)
except ODTExportError, ex:
print >> sys.stderr, ex
print >> sys.stderr, "Conversion failed."
sys.exit(1)
if __name__ == '__main__':
main()