1 """A parser for XML, using the derived class as static DTD."""
3 # Author: Sjoerd Mullender.
9 warnings.warn("The xmllib module is obsolete. Use xml.sax instead.", DeprecationWarning)
14 class Error(RuntimeError):
17 # Regular expressions used for parsing
19 _S = '[ \t\r\n]+' # white space
20 _opS = '[ \t\r\n]*' # optional white space
21 _Name = '[a-zA-Z_:][-a-zA-Z0-9._:]*' # valid XML name
22 _QStr = "(?:'[^']*'|\"[^\"]*\")" # quoted XML string
23 illegal = re.compile('[^\t\r\n -\176\240-\377]') # illegal chars in content
24 interesting = re.compile('[]&<]')
27 ref = re.compile('&(' + _Name + '|#[0-9]+|#x[0-9a-fA-F]+)[^-a-zA-Z0-9._:]')
28 entityref = re.compile('&(?P<name>' + _Name + ')[^-a-zA-Z0-9._:]')
29 charref = re.compile('&#(?P<char>[0-9]+[^0-9]|x[0-9a-fA-F]+[^0-9a-fA-F])')
30 space = re.compile(_S + '$')
31 newline = re.compile('\n')
33 attrfind = re.compile(
34 _S + '(?P<name>' + _Name + ')'
35 '(' + _opS + '=' + _opS +
36 '(?P<value>'+_QStr+'|[-a-zA-Z0-9.:+*%?!\(\)_#=~]+))?')
37 starttagopen = re.compile('<' + _Name)
38 starttagend = re.compile(_opS + '(?P<slash>/?)>')
39 starttagmatch = re.compile('<(?P<tagname>'+_Name+')'
40 '(?P<attrs>(?:'+attrfind.pattern+')*)'+
42 endtagopen = re.compile('</')
43 endbracket = re.compile(_opS + '>')
44 endbracketfind = re.compile('(?:[^>\'"]|'+_QStr+')*>')
45 tagfind = re.compile(_Name)
46 cdataopen = re.compile(r'<!\[CDATA\[')
47 cdataclose = re.compile(r'\]\]>')
48 # this matches one of the following:
49 # SYSTEM SystemLiteral
50 # PUBLIC PubidLiteral SystemLiteral
51 _SystemLiteral = '(?P<%s>'+_QStr+')'
52 _PublicLiteral = '(?P<%s>"[-\'\(\)+,./:=?;!*#@$_%% \n\ra-zA-Z0-9]*"|' \
53 "'[-\(\)+,./:=?;!*#@$_%% \n\ra-zA-Z0-9]*')"
54 _ExternalId = '(?:SYSTEM|' \
55 'PUBLIC'+_S+_PublicLiteral%'pubid'+ \
56 ')'+_S+_SystemLiteral%'syslit'
57 doctype = re.compile('<!DOCTYPE'+_S+'(?P<name>'+_Name+')'
58 '(?:'+_S+_ExternalId+')?'+_opS)
59 xmldecl = re.compile('<\?xml'+_S+
60 'version'+_opS+'='+_opS+'(?P<version>'+_QStr+')'+
61 '(?:'+_S+'encoding'+_opS+'='+_opS+
62 "(?P<encoding>'[A-Za-z][-A-Za-z0-9._]*'|"
63 '"[A-Za-z][-A-Za-z0-9._]*"))?'
64 '(?:'+_S+'standalone'+_opS+'='+_opS+
65 '(?P<standalone>\'(?:yes|no)\'|"(?:yes|no)"))?'+
67 procopen = re.compile(r'<\?(?P<proc>' + _Name + ')' + _opS)
68 procclose = re.compile(_opS + r'\?>')
69 commentopen = re.compile('<!--')
70 commentclose = re.compile('-->')
71 doubledash = re.compile('--')
72 attrtrans = string.maketrans(' \r\n\t', ' ')
74 # definitions for XML namespaces
75 _NCName = '[a-zA-Z_][-a-zA-Z0-9._]*' # XML Name, minus the ":"
76 ncname = re.compile(_NCName + '$')
77 qname = re.compile('(?:(?P<prefix>' + _NCName + '):)?' # optional prefix
78 '(?P<local>' + _NCName + ')$')
80 xmlns = re.compile('xmlns(?::(?P<ncname>'+_NCName+'))?$')
82 # XML parser base class -- find tags and call handler functions.
83 # Usage: p = XMLParser(); p.feed(data); ...; p.close().
84 # The dtd is defined by deriving a class which defines methods with
85 # special names to handle tags: start_foo and end_foo to handle <foo>
86 # and </foo>, respectively. The data between tags is passed to the
87 # parser by calling self.handle_data() with some data as argument (the
88 # data may be split up in arbitrary chunks).
91 attributes = {} # default, to be overridden
92 elements = {} # default, to be overridden
94 # parsing options, settable using keyword args in __init__
95 __accept_unquoted_attributes = 0
96 __accept_missing_endtag_name = 0
99 __translate_attribute_references = 1
101 # Interface -- initialize and reset this instance
102 def __init__(self, **kw):
104 if 'accept_unquoted_attributes' in kw:
105 self.__accept_unquoted_attributes = kw['accept_unquoted_attributes']
106 if 'accept_missing_endtag_name' in kw:
107 self.__accept_missing_endtag_name = kw['accept_missing_endtag_name']
109 self.__map_case = kw['map_case']
110 if 'accept_utf8' in kw:
111 self.__accept_utf8 = kw['accept_utf8']
112 if 'translate_attribute_references' in kw:
113 self.__translate_attribute_references = kw['translate_attribute_references']
116 def __fixelements(self):
119 self.__fixdict(self.__dict__)
120 self.__fixclass(self.__class__)
122 def __fixclass(self, kl):
123 self.__fixdict(kl.__dict__)
124 for k in kl.__bases__:
127 def __fixdict(self, dict):
128 for key in dict.keys():
129 if key[:6] == 'start_':
131 start, end = self.elements.get(tag, (None, None))
133 self.elements[tag] = getattr(self, key), end
134 elif key[:4] == 'end_':
136 start, end = self.elements.get(tag, (None, None))
138 self.elements[tag] = start, getattr(self, key)
140 # Interface -- reset this instance. Loses all unprocessed data
148 self.__seen_doctype = None
149 self.__seen_starttag = 0
150 self.__use_namespaces = 0
151 self.__namespaces = {'xml':None} # xml is implicitly declared
152 # backward compatibility hack: if elements not overridden,
153 # fill it in ourselves
154 if self.elements is XMLParser.elements:
157 # For derived classes only -- enter literal mode (CDATA) till EOF
158 def setnomoretags(self):
159 self.nomoretags = self.literal = 1
161 # For derived classes only -- enter literal mode (CDATA)
162 def setliteral(self, *args):
165 # Interface -- feed some data to the parser. Call this as
166 # often as you want, with as little or as much text as you
167 # want (may include '\n'). (This just saves the text, all the
168 # processing is done by goahead().)
169 def feed(self, data):
170 self.rawdata = self.rawdata + data
173 # Interface -- handle the remaining data
178 # remove self.elements so that we don't leak
181 # Interface -- translate references
182 def translate_references(self, data, all = 1):
183 if not self.__translate_attribute_references:
187 res = amp.search(data, i)
191 res = ref.match(data, s)
193 self.syntax_error("bogus `&'")
201 str = chr(int(str[2:], 16))
203 str = chr(int(str[1:]))
204 if data[i - 1] != ';':
205 self.syntax_error("`;' missing after char reference")
208 if str in self.entitydefs:
209 str = self.entitydefs[str]
211 elif data[i - 1] != ';':
212 self.syntax_error("bogus `&'")
213 i = s + 1 # just past the &
216 self.syntax_error("reference to unknown entity `&%s;'" % str)
217 str = '&' + str + ';'
218 elif data[i - 1] != ';':
219 self.syntax_error("bogus `&'")
220 i = s + 1 # just past the &
223 # when we get here, str contains the translated text and i points
224 # to the end of the string that is to be replaced
225 data = data[:s] + str + data[i:]
231 # Interface - return a dictionary of all namespaces currently valid
232 def getnamespace(self):
234 for t, d, nst in self.stack:
238 # Internal -- handle data as far as reasonable. May leave state
239 # and data to be processed by a subsequent call. If 'end' is
240 # true, force handling all data as if followed by EOF marker.
241 def goahead(self, end):
242 rawdata = self.rawdata
250 self.handle_data(data)
251 self.lineno = self.lineno + data.count('\n')
254 res = interesting.search(rawdata, i)
261 if self.__at_start and space.match(data) is None:
262 self.syntax_error('illegal data at start of file')
264 if not self.stack and space.match(data) is None:
265 self.syntax_error('data not in content')
266 if not self.__accept_utf8 and illegal.search(data):
267 self.syntax_error('illegal character in content')
268 self.handle_data(data)
269 self.lineno = self.lineno + data.count('\n')
272 if rawdata[i] == '<':
273 if starttagopen.match(rawdata, i):
276 self.handle_data(data)
277 self.lineno = self.lineno + data.count('\n')
280 k = self.parse_starttag(i)
282 self.__seen_starttag = 1
283 self.lineno = self.lineno + rawdata[i:k].count('\n')
286 if endtagopen.match(rawdata, i):
287 k = self.parse_endtag(i)
289 self.lineno = self.lineno + rawdata[i:k].count('\n')
292 if commentopen.match(rawdata, i):
295 self.handle_data(data)
296 self.lineno = self.lineno + data.count('\n')
299 k = self.parse_comment(i)
301 self.lineno = self.lineno + rawdata[i:k].count('\n')
304 if cdataopen.match(rawdata, i):
305 k = self.parse_cdata(i)
307 self.lineno = self.lineno + rawdata[i:k].count('\n')
310 res = xmldecl.match(rawdata, i)
312 if not self.__at_start:
313 self.syntax_error("<?xml?> declaration not at start of document")
314 version, encoding, standalone = res.group('version',
317 if version[1:-1] != '1.0':
318 raise Error('only XML version 1.0 supported')
319 if encoding: encoding = encoding[1:-1]
320 if standalone: standalone = standalone[1:-1]
321 self.handle_xml(encoding, standalone)
324 res = procopen.match(rawdata, i)
326 k = self.parse_proc(i)
328 self.lineno = self.lineno + rawdata[i:k].count('\n')
331 res = doctype.match(rawdata, i)
335 self.handle_data(data)
336 self.lineno = self.lineno + data.count('\n')
339 if self.__seen_doctype:
340 self.syntax_error('multiple DOCTYPE elements')
341 if self.__seen_starttag:
342 self.syntax_error('DOCTYPE not at beginning of document')
343 k = self.parse_doctype(res)
345 self.__seen_doctype = res.group('name')
347 self.__seen_doctype = self.__seen_doctype.lower()
348 self.lineno = self.lineno + rawdata[i:k].count('\n')
351 elif rawdata[i] == '&':
354 self.handle_data(data)
357 res = charref.match(rawdata, i)
360 if rawdata[i-1] != ';':
361 self.syntax_error("`;' missing in charref")
364 self.syntax_error('data not in content')
365 self.handle_charref(res.group('char')[:-1])
366 self.lineno = self.lineno + res.group(0).count('\n')
368 res = entityref.match(rawdata, i)
371 if rawdata[i-1] != ';':
372 self.syntax_error("`;' missing in entityref")
374 name = res.group('name')
377 if name in self.entitydefs:
378 self.rawdata = rawdata = rawdata[:res.start(0)] + self.entitydefs[name] + rawdata[i:]
382 self.unknown_entityref(name)
383 self.lineno = self.lineno + res.group(0).count('\n')
385 elif rawdata[i] == ']':
388 self.handle_data(data)
393 if cdataclose.match(rawdata, i):
394 self.syntax_error("bogus `]]>'")
395 self.handle_data(rawdata[i])
399 raise Error('neither < nor & ??')
400 # We get here only if incomplete matches but
408 self.syntax_error("bogus `%s'" % data)
409 if not self.__accept_utf8 and illegal.search(data):
410 self.syntax_error('illegal character in content')
411 self.handle_data(data)
412 self.lineno = self.lineno + data.count('\n')
413 self.rawdata = rawdata[i+1:]
414 return self.goahead(end)
415 self.rawdata = rawdata[i:]
417 if not self.__seen_starttag:
418 self.syntax_error('no elements in file')
420 self.syntax_error('missing end tags')
422 self.finish_endtag(self.stack[-1][0])
424 # Internal -- parse comment, return length or -1 if not terminated
425 def parse_comment(self, i):
426 rawdata = self.rawdata
427 if rawdata[i:i+4] != '<!--':
428 raise Error('unexpected call to handle_comment')
429 res = commentclose.search(rawdata, i+4)
432 if doubledash.search(rawdata, i+4, res.start(0)):
433 self.syntax_error("`--' inside comment")
434 if rawdata[res.start(0)-1] == '-':
435 self.syntax_error('comment cannot end in three dashes')
436 if not self.__accept_utf8 and \
437 illegal.search(rawdata, i+4, res.start(0)):
438 self.syntax_error('illegal character in comment')
439 self.handle_comment(rawdata[i+4: res.start(0)])
442 # Internal -- handle DOCTYPE tag, return length or -1 if not terminated
443 def parse_doctype(self, res):
444 rawdata = self.rawdata
446 name = res.group('name')
449 pubid, syslit = res.group('pubid', 'syslit')
450 if pubid is not None:
451 pubid = pubid[1:-1] # remove quotes
452 pubid = ' '.join(pubid.split()) # normalize
453 if syslit is not None: syslit = syslit[1:-1] # remove quotes
457 if rawdata[k] == '[':
463 if not sq and c == '"':
465 elif not dq and c == "'":
469 elif level <= 0 and c == ']':
470 res = endbracket.match(rawdata, k+1)
473 self.handle_doctype(name, pubid, syslit, rawdata[j+1:k])
480 self.syntax_error("bogus `>' in DOCTYPE")
482 res = endbracketfind.match(rawdata, k)
485 if endbracket.match(rawdata, k) is None:
486 self.syntax_error('garbage in DOCTYPE')
487 self.handle_doctype(name, pubid, syslit, None)
490 # Internal -- handle CDATA tag, return length or -1 if not terminated
491 def parse_cdata(self, i):
492 rawdata = self.rawdata
493 if rawdata[i:i+9] != '<![CDATA[':
494 raise Error('unexpected call to parse_cdata')
495 res = cdataclose.search(rawdata, i+9)
498 if not self.__accept_utf8 and \
499 illegal.search(rawdata, i+9, res.start(0)):
500 self.syntax_error('illegal character in CDATA')
502 self.syntax_error('CDATA not in content')
503 self.handle_cdata(rawdata[i+9:res.start(0)])
506 __xml_namespace_attributes = {'ns':None, 'src':None, 'prefix':None}
507 # Internal -- handle a processing instruction tag
508 def parse_proc(self, i):
509 rawdata = self.rawdata
510 end = procclose.search(rawdata, i)
514 if not self.__accept_utf8 and illegal.search(rawdata, i+2, j):
515 self.syntax_error('illegal character in processing instruction')
516 res = tagfind.match(rawdata, i+2)
518 raise Error('unexpected call to parse_proc')
523 if name == 'xml:namespace':
524 self.syntax_error('old-fashioned namespace declaration')
525 self.__use_namespaces = -1
526 # namespace declaration
527 # this must come after the <?xml?> declaration (if any)
528 # and before the <!DOCTYPE> (if any).
529 if self.__seen_doctype or self.__seen_starttag:
530 self.syntax_error('xml:namespace declaration too late in document')
531 attrdict, namespace, k = self.parse_attributes(name, k, j)
533 self.syntax_error('namespace declaration inside namespace declaration')
534 for attrname in attrdict.keys():
535 if not attrname in self.__xml_namespace_attributes:
536 self.syntax_error("unknown attribute `%s' in xml:namespace tag" % attrname)
537 if not 'ns' in attrdict or not 'prefix' in attrdict:
538 self.syntax_error('xml:namespace without required attributes')
539 prefix = attrdict.get('prefix')
540 if ncname.match(prefix) is None:
541 self.syntax_error('xml:namespace illegal prefix value')
543 if prefix in self.__namespaces:
544 self.syntax_error('xml:namespace prefix not unique')
545 self.__namespaces[prefix] = attrdict['ns']
547 if name.lower() == 'xml':
548 self.syntax_error('illegal processing instruction target name')
549 self.handle_proc(name, rawdata[k:j])
552 # Internal -- parse attributes between i and j
553 def parse_attributes(self, tag, i, j):
554 rawdata = self.rawdata
558 res = attrfind.match(rawdata, i)
561 attrname, attrvalue = res.group('name', 'value')
563 attrname = attrname.lower()
565 if attrvalue is None:
566 self.syntax_error("no value specified for attribute `%s'" % attrname)
568 elif attrvalue[:1] == "'" == attrvalue[-1:] or \
569 attrvalue[:1] == '"' == attrvalue[-1:]:
570 attrvalue = attrvalue[1:-1]
571 elif not self.__accept_unquoted_attributes:
572 self.syntax_error("attribute `%s' value not quoted" % attrname)
573 res = xmlns.match(attrname)
575 # namespace declaration
576 ncname = res.group('ncname')
577 namespace[ncname or ''] = attrvalue or None
578 if not self.__use_namespaces:
579 self.__use_namespaces = len(self.stack)+1
582 self.syntax_error("`<' illegal in attribute value")
583 if attrname in attrdict:
584 self.syntax_error("attribute `%s' specified twice" % attrname)
585 attrvalue = attrvalue.translate(attrtrans)
586 attrdict[attrname] = self.translate_references(attrvalue)
587 return attrdict, namespace, i
589 # Internal -- handle starttag, return length or -1 if not terminated
590 def parse_starttag(self, i):
591 rawdata = self.rawdata
592 # i points to start of tag
593 end = endbracketfind.match(rawdata, i+1)
596 tag = starttagmatch.match(rawdata, i)
597 if tag is None or tag.end(0) != end.end(0):
598 self.syntax_error('garbage in starttag')
600 nstag = tagname = tag.group('tagname')
602 nstag = tagname = nstag.lower()
603 if not self.__seen_starttag and self.__seen_doctype and \
604 tagname != self.__seen_doctype:
605 self.syntax_error('starttag does not match DOCTYPE')
606 if self.__seen_starttag and not self.stack:
607 self.syntax_error('multiple elements on top level')
608 k, j = tag.span('attrs')
609 attrdict, nsdict, k = self.parse_attributes(tagname, k, j)
610 self.stack.append((tagname, nsdict, nstag))
611 if self.__use_namespaces:
612 res = qname.match(tagname)
616 prefix, nstag = res.group('prefix', 'local')
620 for t, d, nst in self.stack:
623 if ns is None and prefix != '':
624 ns = self.__namespaces.get(prefix)
626 nstag = ns + ' ' + nstag
628 nstag = prefix + ':' + nstag # undo split
629 self.stack[-1] = tagname, nsdict, nstag
630 # translate namespace of attributes
631 attrnamemap = {} # map from new name to old name (used for error reporting)
632 for key in attrdict.keys():
633 attrnamemap[key] = key
634 if self.__use_namespaces:
636 for key, val in attrdict.items():
638 res = qname.match(key)
640 aprefix, key = res.group('prefix', 'local')
643 if aprefix is not None:
645 for t, d, nst in self.stack:
649 ans = self.__namespaces.get(aprefix)
651 key = ans + ' ' + key
653 key = aprefix + ':' + key
655 attrnamemap[key] = okey
657 attributes = self.attributes.get(nstag)
658 if attributes is not None:
659 for key in attrdict.keys():
660 if not key in attributes:
661 self.syntax_error("unknown attribute `%s' in tag `%s'" % (attrnamemap[key], tagname))
662 for key, val in attributes.items():
663 if val is not None and not key in attrdict:
665 method = self.elements.get(nstag, (None, None))[0]
666 self.finish_starttag(nstag, attrdict, method)
667 if tag.group('slash') == '/':
668 self.finish_endtag(tagname)
671 # Internal -- parse endtag
672 def parse_endtag(self, i):
673 rawdata = self.rawdata
674 end = endbracketfind.match(rawdata, i+1)
677 res = tagfind.match(rawdata, i+2)
680 self.handle_data(rawdata[i])
682 if not self.__accept_missing_endtag_name:
683 self.syntax_error('no name specified in end tag')
684 tag = self.stack[-1][0]
691 if not self.stack or tag != self.stack[-1][0]:
692 self.handle_data(rawdata[i])
695 if endbracket.match(rawdata, k) is None:
696 self.syntax_error('garbage in end tag')
697 self.finish_endtag(tag)
700 # Internal -- finish processing of start tag
701 def finish_starttag(self, tagname, attrdict, method):
702 if method is not None:
703 self.handle_starttag(tagname, method, attrdict)
705 self.unknown_starttag(tagname, attrdict)
707 # Internal -- finish processing of end tag
708 def finish_endtag(self, tag):
711 self.syntax_error('name-less end tag')
712 found = len(self.stack) - 1
714 self.unknown_endtag(tag)
718 for i in range(len(self.stack)):
719 if tag == self.stack[i][0]:
722 self.syntax_error('unopened end tag')
724 while len(self.stack) > found:
725 if found < len(self.stack) - 1:
726 self.syntax_error('missing close tag for %s' % self.stack[-1][2])
727 nstag = self.stack[-1][2]
728 method = self.elements.get(nstag, (None, None))[1]
729 if method is not None:
730 self.handle_endtag(nstag, method)
732 self.unknown_endtag(nstag)
733 if self.__use_namespaces == len(self.stack):
734 self.__use_namespaces = 0
737 # Overridable -- handle xml processing instruction
738 def handle_xml(self, encoding, standalone):
741 # Overridable -- handle DOCTYPE
742 def handle_doctype(self, tag, pubid, syslit, data):
745 # Overridable -- handle start tag
746 def handle_starttag(self, tag, method, attrs):
749 # Overridable -- handle end tag
750 def handle_endtag(self, tag, method):
753 # Example -- handle character reference, no need to override
754 def handle_charref(self, name):
757 n = int(name[1:], 16)
761 self.unknown_charref(name)
763 if not 0 <= n <= 255:
764 self.unknown_charref(name)
766 self.handle_data(chr(n))
768 # Definition of entities -- derived classes may override
769 entitydefs = {'lt': '<', # must use charref
771 'amp': '&', # must use charref
776 # Example -- handle data, should be overridden
777 def handle_data(self, data):
780 # Example -- handle cdata, could be overridden
781 def handle_cdata(self, data):
784 # Example -- handle comment, could be overridden
785 def handle_comment(self, data):
788 # Example -- handle processing instructions, could be overridden
789 def handle_proc(self, name, data):
792 # Example -- handle relatively harmless syntax errors, could be overridden
793 def syntax_error(self, message):
794 raise Error('Syntax error at line %d: %s' % (self.lineno, message))
796 # To be overridden -- handlers for unknown objects
797 def unknown_starttag(self, tag, attrs): pass
798 def unknown_endtag(self, tag): pass
799 def unknown_charref(self, ref): pass
800 def unknown_entityref(self, name):
801 self.syntax_error("reference to unknown entity `&%s;'" % name)
804 class TestXMLParser(XMLParser):
806 def __init__(self, **kw):
808 XMLParser.__init__(self, **kw)
810 def handle_xml(self, encoding, standalone):
812 print 'xml: encoding =',encoding,'standalone =',standalone
814 def handle_doctype(self, tag, pubid, syslit, data):
816 print 'DOCTYPE:',tag, repr(data)
818 def handle_data(self, data):
819 self.testdata = self.testdata + data
820 if len(repr(self.testdata)) >= 70:
827 print 'data:', repr(data)
829 def handle_cdata(self, data):
831 print 'cdata:', repr(data)
833 def handle_proc(self, name, data):
835 print 'processing:',name,repr(data)
837 def handle_comment(self, data):
841 r = r[:32] + '...' + r[-32:]
844 def syntax_error(self, message):
845 print 'error at line %d:' % self.lineno, message
847 def unknown_starttag(self, tag, attrs):
850 print 'start tag: <' + tag + '>'
852 print 'start tag: <' + tag,
853 for name, value in attrs.items():
854 print name + '=' + '"' + value + '"',
857 def unknown_endtag(self, tag):
859 print 'end tag: </' + tag + '>'
861 def unknown_entityref(self, ref):
863 print '*** unknown entity ref: &' + ref + ';'
865 def unknown_charref(self, ref):
867 print '*** unknown char ref: &#' + ref + ';'
870 XMLParser.close(self)
873 def test(args = None):
875 from time import time
880 opts, args = getopt.getopt(args, 'st')
881 klass = TestXMLParser
904 if f is not sys.stdin:
921 print 'total time: %g' % (t1-t0)
925 print 'total time: %g' % (t1-t0)
928 if __name__ == '__main__':