1 """A parser for HTML and XHTML."""
3 # This file is based on sgmllib.py, but the API is slightly different.
5 # XXX There should be a way to distinguish between PCDATA (parsed
6 # character data -- the normal case), RCDATA (replaceable character
7 # data -- only char and entity references and end tags are special)
8 # and CDATA (character data -- only end tags are special).
14 # Regular expressions used for parsing
16 interesting_normal = re.compile('[&<]')
17 interesting_cdata = re.compile(r'<(/|\Z)')
18 incomplete = re.compile('&[a-zA-Z#]')
20 entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]')
21 charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]')
23 starttagopen = re.compile('<[a-zA-Z]')
24 piclose = re.compile('>')
25 commentclose = re.compile(r'--\s*>')
26 tagfind = re.compile('[a-zA-Z][-.a-zA-Z0-9:_]*')
27 attrfind = re.compile(
28 r'\s*([a-zA-Z_][-.:a-zA-Z_0-9]*)(\s*=\s*'
29 r'(\'[^\']*\'|"[^"]*"|[-a-zA-Z0-9./,:;+*%?!&$\(\)_#=~@]*))?')
31 locatestarttagend = re.compile(r"""
32 <[a-zA-Z][-.a-zA-Z0-9:_]* # tag name
33 (?:\s+ # whitespace before attribute name
34 (?:[a-zA-Z_][-.:a-zA-Z0-9_]* # attribute name
35 (?:\s*=\s* # value indicator
36 (?:'[^']*' # LITA-enclosed value
37 |\"[^\"]*\" # LIT-enclosed value
38 |[^'\">\s]+ # bare value
43 \s* # trailing whitespace
45 endendtag = re.compile('>')
46 endtagfind = re.compile('</\s*([a-zA-Z][-.a-zA-Z0-9:_]*)\s*>')
49 class HTMLParseError(Exception):
50 """Exception raised for all parse errors."""
52 def __init__(self, msg, position=(None, None)):
55 self.lineno = position[0]
56 self.offset = position[1]
60 if self.lineno is not None:
61 result = result + ", at line %d" % self.lineno
62 if self.offset is not None:
63 result = result + ", column %d" % (self.offset + 1)
67 class HTMLParser(markupbase.ParserBase):
68 """Find tags and other markup and call handler functions.
76 Start tags are handled by calling self.handle_starttag() or
77 self.handle_startendtag(); end tags by self.handle_endtag(). The
78 data between tags is passed from the parser to the derived class
79 by calling self.handle_data() with the data as argument (the data
80 may be split up in arbitrary chunks). Entity references are
81 passed by calling self.handle_entityref() with the entity
82 reference as the argument. Numeric character references are
83 passed to self.handle_charref() with the string containing the
84 reference as the argument.
87 CDATA_CONTENT_ELEMENTS = ("script", "style")
91 """Initialize and reset this instance."""
95 """Reset this instance. Loses all unprocessed data."""
98 self.interesting = interesting_normal
99 markupbase.ParserBase.reset(self)
101 def feed(self, data):
102 """Feed data to the parser.
104 Call this as often as you want, with as little or as much text
105 as you want (may include '\n').
107 self.rawdata = self.rawdata + data
111 """Handle any buffered data."""
114 def error(self, message):
115 raise HTMLParseError(message, self.getpos())
117 __starttag_text = None
119 def get_starttag_text(self):
120 """Return full source of start tag: '<...>'."""
121 return self.__starttag_text
123 def set_cdata_mode(self):
124 self.interesting = interesting_cdata
126 def clear_cdata_mode(self):
127 self.interesting = interesting_normal
129 # Internal -- handle data as far as reasonable. May leave state
130 # and data to be processed by a subsequent call. If 'end' is
131 # true, force handling all data as if followed by EOF marker.
132 def goahead(self, end):
133 rawdata = self.rawdata
137 match = self.interesting.search(rawdata, i) # < or &
142 if i < j: self.handle_data(rawdata[i:j])
143 i = self.updatepos(i, j)
145 startswith = rawdata.startswith
146 if startswith('<', i):
147 if starttagopen.match(rawdata, i): # < + letter
148 k = self.parse_starttag(i)
149 elif startswith("</", i):
150 k = self.parse_endtag(i)
151 elif startswith("<!--", i):
152 k = self.parse_comment(i)
153 elif startswith("<?", i):
155 elif startswith("<!", i):
156 k = self.parse_declaration(i)
158 self.handle_data("<")
164 self.error("EOF in middle of construct")
166 i = self.updatepos(i, k)
167 elif startswith("&#", i):
168 match = charref.match(rawdata, i)
170 name = match.group()[2:-1]
171 self.handle_charref(name)
173 if not startswith(';', k-1):
175 i = self.updatepos(i, k)
179 elif startswith('&', i):
180 match = entityref.match(rawdata, i)
182 name = match.group(1)
183 self.handle_entityref(name)
185 if not startswith(';', k-1):
187 i = self.updatepos(i, k)
189 match = incomplete.match(rawdata, i)
191 # match.group() will contain at least 2 chars
192 if end and match.group() == rawdata[i:]:
193 self.error("EOF in middle of entity or char ref")
197 # not the end of the buffer, and can't be confused
198 # with some other construct
199 self.handle_data("&")
200 i = self.updatepos(i, i + 1)
204 assert 0, "interesting.search() lied"
207 self.handle_data(rawdata[i:n])
208 i = self.updatepos(i, n)
209 self.rawdata = rawdata[i:]
211 # Internal -- parse processing instr, return end or -1 if not terminated
212 def parse_pi(self, i):
213 rawdata = self.rawdata
214 assert rawdata[i:i+2] == '<?', 'unexpected call to parse_pi()'
215 match = piclose.search(rawdata, i+2) # >
219 self.handle_pi(rawdata[i+2: j])
223 # Internal -- handle starttag, return end or -1 if not terminated
224 def parse_starttag(self, i):
225 self.__starttag_text = None
226 endpos = self.check_for_whole_start_tag(i)
229 rawdata = self.rawdata
230 self.__starttag_text = rawdata[i:endpos]
232 # Now parse the data between i+1 and j into a tag and attrs
234 match = tagfind.match(rawdata, i+1)
235 assert match, 'unexpected call to parse_starttag()'
237 self.lasttag = tag = rawdata[i+1:k].lower()
240 m = attrfind.match(rawdata, k)
243 attrname, rest, attrvalue = m.group(1, 2, 3)
246 elif attrvalue[:1] == '\'' == attrvalue[-1:] or \
247 attrvalue[:1] == '"' == attrvalue[-1:]:
248 attrvalue = attrvalue[1:-1]
249 attrvalue = self.unescape(attrvalue)
250 attrs.append((attrname.lower(), attrvalue))
253 end = rawdata[k:endpos].strip()
254 if end not in (">", "/>"):
255 lineno, offset = self.getpos()
256 if "\n" in self.__starttag_text:
257 lineno = lineno + self.__starttag_text.count("\n")
258 offset = len(self.__starttag_text) \
259 - self.__starttag_text.rfind("\n")
261 offset = offset + len(self.__starttag_text)
262 self.error("junk characters in start tag: %r"
263 % (rawdata[k:endpos][:20],))
264 if end.endswith('/>'):
265 # XHTML-style empty tag: <span attr="value" />
266 self.handle_startendtag(tag, attrs)
268 self.handle_starttag(tag, attrs)
269 if tag in self.CDATA_CONTENT_ELEMENTS:
270 self.set_cdata_mode()
273 # Internal -- check to see if we have a complete starttag; return end
274 # or -1 if incomplete.
275 def check_for_whole_start_tag(self, i):
276 rawdata = self.rawdata
277 m = locatestarttagend.match(rawdata, i)
280 next = rawdata[j:j+1]
284 if rawdata.startswith("/>", j):
286 if rawdata.startswith("/", j):
290 self.updatepos(i, j + 1)
291 self.error("malformed empty start tag")
295 if next in ("abcdefghijklmnopqrstuvwxyz=/"
296 "ABCDEFGHIJKLMNOPQRSTUVWXYZ"):
297 # end of input in or before attribute value, or we have the
298 # '/' from a '/>' ending
301 self.error("malformed start tag")
302 raise AssertionError("we should not get here!")
304 # Internal -- parse endtag, return end or -1 if incomplete
305 def parse_endtag(self, i):
306 rawdata = self.rawdata
307 assert rawdata[i:i+2] == "</", "unexpected call to parse_endtag"
308 match = endendtag.search(rawdata, i+1) # >
312 match = endtagfind.match(rawdata, i) # </ + tag + >
314 self.error("bad end tag: %r" % (rawdata[i:j],))
316 self.handle_endtag(tag.lower())
317 self.clear_cdata_mode()
320 # Overridable -- finish processing of start+end tag: <tag.../>
321 def handle_startendtag(self, tag, attrs):
322 self.handle_starttag(tag, attrs)
323 self.handle_endtag(tag)
325 # Overridable -- handle start tag
326 def handle_starttag(self, tag, attrs):
329 # Overridable -- handle end tag
330 def handle_endtag(self, tag):
333 # Overridable -- handle character reference
334 def handle_charref(self, name):
337 # Overridable -- handle entity reference
338 def handle_entityref(self, name):
341 # Overridable -- handle data
342 def handle_data(self, data):
345 # Overridable -- handle comment
346 def handle_comment(self, data):
349 # Overridable -- handle declaration
350 def handle_decl(self, decl):
353 # Overridable -- handle processing instruction
354 def handle_pi(self, data):
357 def unknown_decl(self, data):
358 self.error("unknown declaration: %r" % (data,))
360 # Internal -- helper to remove special character quoting
361 def unescape(self, s):
364 s = s.replace("<", "<")
365 s = s.replace(">", ">")
366 s = s.replace("'", "'")
367 s = s.replace(""", '"')
368 s = s.replace("&", "&") # Must be last