root / atomisator / sources / feedparser.py

# $Id: feedparser.py 216 2006-11-12 11:36:49Z tarek $
"""Universal feed parser

Handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom feeds

Visit http://feedparser.org/ for the latest version
Visit http://feedparser.org/docs/ for the latest documentation

Required: Python 2.1 or later
Recommended: Python 2.3 or later
Recommended: CJKCodecs and iconv_codec <http://cjkpython.i18n.org/>
"""

#__version__ = "pre-3.3-" + "$Revision: 1.51 $"[11:15] + "-cvs"
__version__ = "3.3"
__license__ = "Python"
__copyright__ = "Copyright 2002-4, Mark Pilgrim"
__author__ = "Mark Pilgrim <http://diveintomark.org/>"
__contributors__ = ["Jason Diamond <http://injektilo.org/>",
                    "John Beimler <http://john.beimler.org/>",
                    "Fazal Majid <http://www.majid.info/mylos/weblog/>",
                    "Aaron Swartz <http://aaronsw.com>"]
_debug = 0

# HTTP "User-Agent" header to send to servers when downloading feeds.
# If you are embedding feedparser in a larger application, you should
# change this to your application name and URL.
USER_AGENT = "UniversalFeedParser/%s +http://feedparser.org/" % __version__

# HTTP "Accept" header to send to servers when downloading feeds.  If you don't
# want to send an Accept header, set this to None.
ACCEPT_HEADER = "application/atom+xml,application/rdf+xml,application/rss+xml,application/x-netcdf,application/xml;q=0.9,text/xml;q=0.2,*/*;q=0.1"

# List of preferred XML parsers, by SAX driver name.  These will be tried first,
# but if they're not installed, Python will keep searching through its own list
# of pre-installed parsers until it finds one that supports everything we need.
PREFERRED_XML_PARSERS = ["drv_libxml2"]

# If you want feedparser to automatically run HTML markup through HTML Tidy, set
# this to 1.  This is off by default because of reports of crashing on some
# platforms.  If it crashes for you, please submit a bug report with your OS
# platform, Python version, and the URL of the feed you were attempting to parse.
# Requires mxTidy <http://www.egenix.com/files/python/mxTidy.html>
TIDY_MARKUP = 0

# ---------- required modules (should come with any Python distribution) ----------
import sgmllib, re, sys, copy, urlparse, time, rfc822, types, cgi
try:
    from cStringIO import StringIO as _StringIO
except:
    from StringIO import StringIO as _StringIO

# ---------- optional modules (feedparser will work without these, but with reduced functionality) ----------

# gzip is included with most Python distributions, but may not be available if you compiled your own
try:
    import gzip
except:
    gzip = None
try:
    import zlib
except:
    zlib = None

# timeoutsocket allows feedparser to time out rather than hang forever on ultra-slow servers.
# Python 2.3 now has this functionality available in the standard socket library, so under
# 2.3 you don't need to install anything.  But you probably should anyway, because the socket
# module is buggy and timeoutsocket is better.
try:
    import timeoutsocket # http://www.timo-tasi.org/python/timeoutsocket.py
    timeoutsocket.setDefaultSocketTimeout(20)
except ImportError:
    import socket
    if hasattr(socket, 'setdefaulttimeout'):
        socket.setdefaulttimeout(20)
import urllib, urllib2

_mxtidy = None
if TIDY_MARKUP:
    try:
        from mx.Tidy import Tidy as _mxtidy
    except:
        pass

# If a real XML parser is available, feedparser will attempt to use it.  feedparser has
# been tested with the built-in SAX parser, PyXML, and libxml2.  On platforms where the
# Python distribution does not come with an XML parser (such as Mac OS X 10.2 and some
# versions of FreeBSD), feedparser will quietly fall back on regex-based parsing.
try:
    import xml.sax
    xml.sax.make_parser(PREFERRED_XML_PARSERS) # test for valid parsers
    from xml.sax.saxutils import escape as _xmlescape
    _XML_AVAILABLE = 1
except:
    _XML_AVAILABLE = 0
    def _xmlescape(data):
        data = data.replace("&", "&amp;")
        data = data.replace(">", "&gt;")
        data = data.replace("<", "&lt;")
        return data

# base64 support for Atom feeds that contain embedded binary data
try:
    import base64, binascii
except:
    base64 = binascii = None

# cjkcodecs and iconv_codec provide support for more character encodings.
# Both are available from http://cjkpython.i18n.org/
try:
    import cjkcodecs.aliases
except:
    pass
try:
    import iconv_codec
except:
    pass

# ---------- don't touch these ----------
class CharacterEncodingOverride(Exception): pass
class CharacterEncodingUnknown(Exception): pass
class NonXMLContentType(Exception): pass

sgmllib.tagfind = re.compile('[a-zA-Z][-_.:a-zA-Z0-9]*')
sgmllib.special = re.compile('<!')
sgmllib.charref = re.compile('&#(x?[0-9A-Fa-f]+)[^0-9A-Fa-f]')

SUPPORTED_VERSIONS = {'': 'unknown',
                      'rss090': 'RSS 0.90',
                      'rss091n': 'RSS 0.91 (Netscape)',
                      'rss091u': 'RSS 0.91 (Userland)',
                      'rss092': 'RSS 0.92',
                      'rss093': 'RSS 0.93',
                      'rss094': 'RSS 0.94',
                      'rss20': 'RSS 2.0',
                      'rss10': 'RSS 1.0',
                      'rss': 'RSS (unknown version)',
                      'atom01': 'Atom 0.1',
                      'atom02': 'Atom 0.2',
                      'atom03': 'Atom 0.3',
                      'atom': 'Atom (unknown version)',
                      'cdf': 'CDF',
                      'hotrss': 'Hot RSS'
                      }

try:
    UserDict = dict
except NameError:
    # Python 2.1 does not have dict
    from UserDict import UserDict
    def dict(aList):
        rc = {}
        for k, v in aList:
            rc[k] = v
        return rc

class FeedParserDict(UserDict):
    def __getitem__(self, key):
        keymap = {'channel': 'feed',
                  'items': 'entries',
                  'guid': 'id',
                  'date': 'modified',
                  'date_parsed': 'modified_parsed',
                  'description': ['tagline', 'summary']}
        realkey = keymap.get(key, key)
        if type(realkey) == types.ListType:
            for k in realkey:
                if UserDict.has_key(self, k):
                    return UserDict.__getitem__(self, k)
            return UserDict.__getitem__(self, key)
        return UserDict.__getitem__(self, realkey)

    def has_key(self, key):
        return hasattr(self, key) or UserDict.has_key(self, key)

    def __getattr__(self, key):
        try:
            return self.__dict__[key]
        except KeyError:
            pass
        try:
            return self.__getitem__(key)
        except:
            raise AttributeError, "object has no attribute '%s'" % key

    def __contains__(self, key):
        return self.has_key(key)

def zopeCompatibilityHack():
    global FeedParserDict
    del FeedParserDict
    def FeedParserDict(aDict=None):
        rc = {}
        if aDict:
            rc.update(aDict)
        return rc

_ebcdic_to_ascii_map = None
def _ebcdic_to_ascii(s):
    global _ebcdic_to_ascii_map
    if not _ebcdic_to_ascii_map:
        emap = (
            0,1,2,3,156,9,134,127,151,141,142,11,12,13,14,15,
            16,17,18,19,157,133,8,135,24,25,146,143,28,29,30,31,
            128,129,130,131,132,10,23,27,136,137,138,139,140,5,6,7,
            144,145,22,147,148,149,150,4,152,153,154,155,20,21,158,26,
            32,160,161,162,163,164,165,166,167,168,91,46,60,40,43,33,
            38,169,170,171,172,173,174,175,176,177,93,36,42,41,59,94,
            45,47,178,179,180,181,182,183,184,185,124,44,37,95,62,63,
            186,187,188,189,190,191,192,193,194,96,58,35,64,39,61,34,
            195,97,98,99,100,101,102,103,104,105,196,197,198,199,200,201,
            202,106,107,108,109,110,111,112,113,114,203,204,205,206,207,208,
            209,126,115,116,117,118,119,120,121,122,210,211,212,213,214,215,
            216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,
            123,65,66,67,68,69,70,71,72,73,232,233,234,235,236,237,
            125,74,75,76,77,78,79,80,81,82,238,239,240,241,242,243,
            92,159,83,84,85,86,87,88,89,90,244,245,246,247,248,249,
            48,49,50,51,52,53,54,55,56,57,250,251,252,253,254,255
            )
        import string
        _ebcdic_to_ascii_map = string.maketrans( \
            "".join(map(chr, range(256))), "".join(map(chr, emap)))
    return s.translate(_ebcdic_to_ascii_map)

class _FeedParserMixin:
    namespaces = {"": "",
                  "http://backend.userland.com/rss": "",
                  "http://blogs.law.harvard.edu/tech/rss": "",
                  "http://purl.org/rss/1.0/": "",
                  "http://my.netscape.com/rdf/simple/0.9/": "",
                  "http://example.com/newformat#": "",
                  "http://example.com/necho": "",
                  "http://purl.org/echo/": "",
                  "uri/of/echo/namespace#": "",
                  "http://purl.org/pie/": "",
                  "http://purl.org/atom/ns#": "",
                  "http://purl.org/rss/1.0/modules/rss091#": "",

                  "http://webns.net/mvcb/":                               "admin",
                  "http://purl.org/rss/1.0/modules/aggregation/":         "ag",
                  "http://purl.org/rss/1.0/modules/annotate/":            "annotate",
                  "http://media.tangent.org/rss/1.0/":                    "audio",
                  "http://backend.userland.com/blogChannelModule":        "blogChannel",
                  "http://web.resource.org/cc/":                          "cc",
                  "http://backend.userland.com/creativeCommonsRssModule": "creativeCommons",
                  "http://purl.org/rss/1.0/modules/company":              "co",
                  "http://purl.org/rss/1.0/modules/content/":             "content",
                  "http://my.theinfo.org/changed/1.0/rss/":               "cp",
                  "http://purl.org/dc/elements/1.1/":                     "dc",
                  "http://purl.org/dc/terms/":                            "dcterms",
                  "http://purl.org/rss/1.0/modules/email/":               "email",
                  "http://purl.org/rss/1.0/modules/event/":               "ev",
                  "http://postneo.com/icbm/":                             "icbm",
                  "http://purl.org/rss/1.0/modules/image/":               "image",
                  "http://xmlns.com/foaf/0.1/":                           "foaf",
                  "http://freshmeat.net/rss/fm/":                         "fm",
                  "http://purl.org/rss/1.0/modules/link/":                "l",
                  "http://madskills.com/public/xml/rss/module/pingback/": "pingback",
                  "http://prismstandard.org/namespaces/1.2/basic/":       "prism",
                  "http://www.w3.org/1999/02/22-rdf-syntax-ns#":          "rdf",
                  "http://www.w3.org/2000/01/rdf-schema#":                "rdfs",
                  "http://purl.org/rss/1.0/modules/reference/":           "ref",
                  "http://purl.org/rss/1.0/modules/richequiv/":           "reqv",
                  "http://purl.org/rss/1.0/modules/search/":              "search",
                  "http://purl.org/rss/1.0/modules/slash/":               "slash",
                  "http://purl.org/rss/1.0/modules/servicestatus/":       "ss",
                  "http://hacks.benhammersley.com/rss/streaming/":        "str",
                  "http://purl.org/rss/1.0/modules/subscription/":        "sub",
                  "http://purl.org/rss/1.0/modules/syndication/":         "sy",
                  "http://purl.org/rss/1.0/modules/taxonomy/":            "taxo",
                  "http://purl.org/rss/1.0/modules/threading/":           "thr",
                  "http://purl.org/rss/1.0/modules/textinput/":           "ti",
                  "http://madskills.com/public/xml/rss/module/trackback/":"trackback",
                  "http://wellformedweb.org/CommentAPI/":                 "wfw",
                  "http://purl.org/rss/1.0/modules/wiki/":                "wiki",
                  "http://schemas.xmlsoap.org/soap/envelope/":            "soap",
                  "http://www.w3.org/1999/xhtml":                         "xhtml",
                  "http://www.w3.org/XML/1998/namespace":                 "xml"
}

    can_be_relative_uri = ['link', 'id', 'wfw_comment', 'wfw_commentrss', 'docs', 'url', 'comments', 'license']
    can_contain_relative_uris = ['content', 'title', 'summary', 'info', 'tagline', 'copyright', 'description']
    can_contain_dangerous_markup = ['content', 'title', 'summary', 'info', 'tagline', 'copyright', 'description']
    html_types = ['text/html', 'application/xhtml+xml']

    def __init__(self, baseuri=None, baselang=None, encoding='utf-8'):
        if _debug: sys.stderr.write("initializing FeedParser\n")
        self.feeddata = FeedParserDict() # feed-level data
        self.encoding = encoding # character encoding
        self.entries = [] # list of entry-level data
        self.version = '' # feed type/version, see SUPPORTED_VERSIONS

        # the following are used internally to track state;
        # some of this is kind of out of control and should
        # probably be refactored into a finite state machine
        self.infeed = 0
        self.inentry = 0
        self.incontent = 0
        self.intextinput = 0
        self.inimage = 0
        self.inauthor = 0
        self.incontributor = 0
        self.contentparams = FeedParserDict()
        self.namespacemap = {}
        self.elementstack = []
        self.basestack = []
        self.langstack = []
        self.baseuri = baseuri or ''
        self.lang = baselang or None
        if baselang:
            self.feeddata['language'] = baselang

    def unknown_starttag(self, tag, attrs):
        if _debug: sys.stderr.write('start %s with %s\n' % (tag, attrs))
        # normalize attrs
        attrs = [(k.lower(), v) for k, v in attrs]
        attrs = [(k, k in ('rel', 'type') and v.lower() or v) for k, v in attrs]

        # track xml:base and xml:lang
        attrsD = dict(attrs)
        baseuri = attrsD.get('xml:base', attrsD.get('base')) or self.baseuri
        self.baseuri = baseuri
        lang = attrsD.get('xml:lang', attrsD.get('lang'))
        if lang == '':
            # xml:lang could be explicitly set to '', we need to capture that
            lang = None
        elif lang is None:
            # if no xml:lang is specified, use parent lang
            lang = self.lang
        if lang:
            if tag in ('feed', 'rss', 'rdf:RDF'):
                self.feeddata['language'] = lang
        self.lang = lang
        self.basestack.append(baseuri)
        self.langstack.append(lang)

        # track namespaces
        for prefix, uri in attrs:
            if prefix.startswith('xmlns:'):
                self.trackNamespace(prefix[6:], uri)
            elif prefix == 'xmlns':
                self.trackNamespace(None, uri)

        # track inline content
        if self.incontent and self.contentparams.get('mode') == 'escaped':
            # element declared itself as escaped markup, but it isn't really
            self.contentparams['mode'] = 'xml'
        if self.incontent and self.contentparams.get('mode') == 'xml':
            # Note: probably shouldn't simply recreate localname here, but
            # our namespace handling isn't actually 100% correct in cases where
            # the feed redefines the default namespace (which is actually
            # the usual case for inline content, thanks Sam), so here we
            # cheat and just reconstruct the element based on localname
            # because that compensates for the bugs in our namespace handling.
            # This will horribly munge inline content with non-empty qnames,
            # but nobody actually does that, so I'm not fixing it.
            tag = tag.split(':')[-1]
            return self.handle_data("<%s%s>" % (tag, "".join([' %s="