lxml.html

1 import threading 2 import re 3 import urlparse 4 import copy 5 from lxml import etree 6 from lxml.html import defs 7 from lxml import cssselect 8 from lxml.html.setmixin import SetMixin 9 try: 10 from UserDict import DictMixin 11 except ImportError: 12 # DictMixin was introduced in Python 2.4 13 from lxml.html._dictmixin import DictMixin 14 import sets 15 16 __all__ = [ 17 'document_fromstring', 'fragment_fromstring', 'fragments_fromstring', 'fromstring', 18 'tostring', 'Element', 'defs', 'open_in_browser', 'submit_form', 19 'find_rel_links', 'find_class', 'make_links_absolute', 20 'resolve_base_href', 'iterlinks', 'rewrite_links', 'open_in_browser'] 21 22 _rel_links_xpath = etree.XPath("descendant-or-self::a[@rel]") 23 #_class_xpath = etree.XPath(r"descendant-or-self::*[regexp:match(@class, concat('\b', $class_name, '\b'))]", {'regexp': 'http://exslt.org/regular-expressions'}) 24 _class_xpath = etree.XPath("descendant-or-self::*[@class and contains(concat(' ', normalize-space(@class), ' '), concat(' ', $class_name, ' '))]") 25 _id_xpath = etree.XPath("descendant-or-self::*[@id=$id]") 26 _collect_string_content = etree.XPath("string()") 27 _css_url_re = re.compile(r'url$(.*?)$', re.I) 28 _css_import_re = re.compile(r'@import "(.*?)"') 29 _label_xpath = etree.XPath("//label[@for=$id]") 30 _archive_re = re.compile(r'[^ ]+') 31

32 -class HtmlMixin(object):

33

34 - def base_url(self):

35 """ 36 Returns the base URL, given when the page was parsed. 37 38 Use with ``urlparse.urljoin(el.base_url, href)`` to get 39 absolute URLs. 40 """ 41 return self.getroottree().docinfo.URL

42 base_url = property(base_url, doc=base_url.__doc__) 43

44 - def forms(self):

45 """ 46 Return a list of all the forms 47 """ 48 return list(self.getiterator('form'))

49 forms = property(forms, doc=forms.__doc__) 50

51 - def body(self):

52 """ 53 Return the <body> element. Can be called from a child element 54 to get the document's head. 55 """ 56 return self.xpath('//body')[0]

57 body = property(body, doc=body.__doc__) 58

59 - def head(self):

60 """ 61 Returns the <head> element. Can be called from a child 62 element to get the document's head. 63 """ 64 return self.xpath('//head')[0]

65 head = property(head, doc=head.__doc__) 66

67 - def label__get(self):

68 """ 69 Get or set any <label> element associated with this element. 70 """ 71 id = self.get('id') 72 if not id: 73 return None 74 result = _label_xpath(self, id=id) 75 if not result: 76 return None 77 else: 78 return result[0]

79 - def label__set(self, label):

80 id = self.get('id') 81 if not id: 82 raise TypeError( 83 "You cannot set a label for an element (%r) that has no id" 84 % self) 85 if not label.tag == 'label': 86 raise TypeError( 87 "You can only assign label to a label element (not %r)" 88 % label) 89 label.set('for', id)

90 - def label__del(self):

91 label = self.label 92 if label is not None: 93 del label.attrib['for']

94 label = property(label__get, label__set, label__del, doc=label__get.__doc__) 95

96 - def drop_tree(self):

97 """ 98 Removes this element from the tree, including its children and 99 text. The tail text is joined to the previous element or 100 parent. 101 """ 102 parent = self.getparent() 103 assert parent is not None 104 if self.tail: 105 previous = self.getprevious() 106 if previous is None: 107 parent.text = (parent.text or '') + self.tail 108 else: 109 previous.tail = (previous.tail or '') + self.tail 110 parent.remove(self)

111

112 - def drop_tag(self):

113 """ 114 Remove the tag, but not its children or text. The children and text 115 are merged into the parent. 116 117 Example:: 118 119 >>> h = fragment_fromstring('<div>Hello <b>World!</b></div>') 120 >>> h.find('//b').drop_tag() 121 >>> print tostring(h) 122 <div>Hello World!</div> 123 """ 124 parent = self.getparent() 125 assert parent is not None 126 previous = self.getprevious() 127 if self.text and isinstance(self.tag, basestring): 128 # not a Comment, etc. 129 if previous is None: 130 parent.text = (parent.text or '') + self.text 131 else: 132 previous.tail = (previous.tail or '') + self.text 133 if self.tail: 134 if len(self): 135 last = self[-1] 136 last.tail = (last.tail or '') + self.tail 137 elif previous is None: 138 parent.text = (parent.text or '') + self.tail 139 else: 140 previous.tail = (previous.tail or '') + self.tail 141 index = parent.index(self) 142 parent[index:index+1] = self[:]

143

144 - def find_rel_links(self, rel):

145 """ 146 Find any links like ``<a rel="{rel}">...</a>``; returns a list of elements. 147 """ 148 rel = rel.lower() 149 return [el for el in _rel_links_xpath(self) 150 if el.get('rel').lower() == rel]

151

152 - def find_class(self, class_name):

153 """ 154 Find any elements with the given class name. 155 """ 156 return _class_xpath(self, class_name=class_name)

157

158 - def get_element_by_id(self, id, *default):

159 """ 160 Get the first element in a document with the given id. If none is 161 found, return the default argument if provided or raise KeyError 162 otherwise. 163 164 Note that there can be more than one element with the same id, 165 and this isn't uncommon in HTML documents found in the wild. 166 Browsers return only the first match, and this function does 167 the same. 168 """ 169 try: 170 # FIXME: should this check for multiple matches? 171 # browsers just return the first one 172 return _id_xpath(self, id=id)[0] 173 except IndexError: 174 if default: 175 return default[0] 176 else: 177 raise KeyError, id

178

179 - def text_content(self):

180 """ 181 Return the text content of the tag (and the text in any children). 182 """ 183 return _collect_string_content(self)

184

185 - def cssselect(self, expr):

186 """ 187 Run the CSS expression on this element and its children, 188 returning a list of the results. 189 190 Equivalent to lxml.cssselect.CSSSelect(expr)(self) -- note 191 that pre-compiling the expression can provide a substantial 192 speedup. 193 """ 194 return cssselect.CSSSelect(expr)(self)

195 196 ######################################## 197 ## Link functions 198 ######################################## 199

200 - def make_links_absolute(self, base_url=None, resolve_base_href=True):

201 """ 202 Make all links in the document absolute, given the 203 ``base_url`` for the document (the full URL where the document 204 came from), or if no ``base_url`` is given, then the ``.base_url`` of the document. 205 206 If ``resolve_base_href`` is true, then any ``<base href>`` 207 tags in the document are used *and* removed from the document. 208 If it is false then any such tag is ignored. 209 """ 210 if base_url is None: 211 base_url = self.base_url 212 if base_url is None: 213 raise TypeError( 214 "No base_url given, and the document has no base_url") 215 if resolve_base_href: 216 self.resolve_base_href() 217 def link_repl(href): 218 return urlparse.urljoin(base_url, href)

219 self.rewrite_links(link_repl)

220

221 - def resolve_base_href(self):

222 """ 223 Find any ``<base href>`` tag in the document, and apply its 224 values to all links found in the document. Also remove the 225 tag once it has been applied. 226 """ 227 base_href = None 228 basetags = self.xpath('//base[@href]') 229 for b in basetags: 230 base_href = b.get('href') 231 b.drop_tree() 232 if not base_href: 233 return 234 self.make_links_absolute(base_href, resolve_base_href=False)

235

236 - def iterlinks(self):

237 """ 238 Yield (element, attribute, link, pos), where attribute may be None 239 (indicating the link is in the text). ``pos`` is the position 240 where the link occurs; often 0, but sometimes something else in 241 the case of links in stylesheets or style tags. 242 243 Note: <base href> is *not* taken into account in any way. The 244 link you get is exactly the link in the document. 245 """ 246 link_attrs = defs.link_attrs 247 for el in self.getiterator(): 248 attribs = el.attrib 249 if el.tag != 'object': 250 for attrib in link_attrs: 251 if attrib in attribs: 252 yield (el, attrib, attribs[attrib], 0) 253 elif el.tag == 'object': 254 codebase = None 255 ## <object> tags have attributes that are relative to 256 ## codebase 257 if 'codebase' in attribs: 258 codebase = el.get('codebase') 259 yield (el, 'codebase', codebase, 0) 260 for attrib in 'classid', 'data': 261 if attrib in attribs: 262 value = el.get(attrib) 263 if codebase is not None: 264 value = urlparse.urljoin(codebase, value) 265 yield (el, attrib, value, 0) 266 if 'archive' in attribs: 267 for match in _archive_re.finditer(el.get('archive')): 268 value = match.group(0) 269 if codebase is not None: 270 value = urlparse.urljoin(codebase, value) 271 yield (el, 'archive', value, match.start()) 272 if el.tag == 'param': 273 valuetype = el.get('valuetype') or '' 274 if valuetype.lower() == 'ref': 275 ## FIXME: while it's fine we *find* this link, 276 ## according to the spec we aren't supposed to 277 ## actually change the value, including resolving 278 ## it. It can also still be a link, even if it 279 ## doesn't have a valuetype="ref" (which seems to be the norm) 280 ## http://www.w3.org/TR/html401/struct/objects.html#adef-valuetype 281 yield (el, 'value', el.get('value'), 0) 282 if el.tag == 'style' and el.text: 283 for match in _css_url_re.finditer(el.text): 284 yield (el, None, match.group(1), match.start(1)) 285 for match in _css_import_re.finditer(el.text): 286 yield (el, None, match.group(1), match.start(1)) 287 if 'style' in attribs: 288 for match in _css_url_re.finditer(attribs['style']): 289 yield (el, 'style', match.group(1), match.start(1))

290

291 - def rewrite_links(self, link_repl_func, resolve_base_href=True, 292 base_href=None):

293 """ 294 Rewrite all the links in the document. For each link 295 ``link_repl_func(link)`` will be called, and the return value 296 will replace the old link. 297 298 Note that links may not be absolute (unless you first called 299 ``make_links_absolute()``), and may be internal (e.g., 300 ``'#anchor'``). They can also be values like 301 ``'mailto:email'`` or ``'javascript:expr'``. 302 303 If you give ``base_href`` then all links passed to 304 ``link_repl_func()`` will take that into account. 305 306 If the ``link_repl_func`` returns None, the attribute or 307 tag text will be removed completely. 308 """ 309 if base_href is not None: 310 # FIXME: this can be done in one pass with a wrapper 311 # around link_repl_func 312 self.make_links_absolute(base_href, resolve_base_href=resolve_base_href) 313 elif resolve_base_href: 314 self.resolve_base_href() 315 for el, attrib, link, pos in self.iterlinks(): 316 new_link = link_repl_func(link) 317 if new_link == link: 318 continue 319 if new_link is None: 320 # Remove the attribute or element content 321 if attrib is None: 322 el.text = '' 323 else: 324 del el.attrib[attrib] 325 continue 326 if attrib is None: 327 new = el.text[:pos] + new_link + el.text[pos+len(link):] 328 el.text = new 329 else: 330 cur = el.attrib[attrib] 331 if not pos and len(cur) == len(link): 332 # Most common case 333 el.attrib[attrib] = new_link 334 else: 335 new = cur[:pos] + new_link + cur[pos+len(link):] 336 el.attrib[attrib] = new

337 338

339 -class _MethodFunc(object):

340 """ 341 An object that represents a method on an element as a function; 342 the function takes either an element or an HTML string. It 343 returns whatever the function normally returns, or if the function 344 works in-place (and so returns None) it returns a serialized form 345 of the resulting document. 346 """

347 - def __init__(self, name, copy=False, source_class=HtmlMixin):

348 self.name = name 349 self.copy = copy 350 self.__doc__ = getattr(source_class, self.name).__doc__

351 - def __call__(self, doc, *args, **kw):

352 if isinstance(doc, basestring): 353 if 'copy' in kw: 354 raise TypeError( 355 "The keyword 'copy' can only be used with element inputs to %s, not a string input" % self.name) 356 return_string = True 357 doc = fromstring(doc, **kw) 358 else: 359 if 'copy' in kw: 360 copy = kw.pop('copy') 361 else: 362 copy = self.copy 363 return_string = False 364 if copy: 365 doc = copy.deepcopy(doc) 366 meth = getattr(doc, self.name) 367 result = meth(*args, **kw) 368 # FIXME: this None test is a bit sloppy 369 if result is None: 370 # Then return what we got in 371 if return_string: 372 return tostring(doc) 373 else: 374 return doc 375 else: 376 return result

377 378 find_rel_links = _MethodFunc('find_rel_links', copy=False) 379 find_class = _MethodFunc('find_class', copy=False) 380 make_links_absolute = _MethodFunc('make_links_absolute', copy=True) 381 resolve_base_href = _MethodFunc('resolve_base_href', copy=True) 382 iterlinks = _MethodFunc('iterlinks', copy=False) 383 rewrite_links = _MethodFunc('rewrite_links', copy=True) 384

385 -class HtmlComment(etree.CommentBase, HtmlMixin):

386 pass

387

388 -class HtmlElement(etree.ElementBase, HtmlMixin):

389 pass

390

391 -class HtmlProcessingInstruction(etree.PIBase, HtmlMixin):

392 pass

393

394 -class HtmlEntity(etree.EntityBase, HtmlMixin):

395 pass

396 397

398 -class HtmlElementClassLookup(etree.CustomElementClassLookup):

399 """A lookup scheme for HTML Element classes. 400 401 To create a lookup instance with different Element classes, pass a tag 402 name mapping of Element classes in the ``classes`` keyword argument and/or 403 a tag name mapping of Mixin classes in the ``mixins`` keyword argument. 404 The special key '*' denotes a Mixin class that should be mixed into all 405 Element classes. 406 """ 407 _default_element_classes = {} 408

409 - def __init__(self, classes=None, mixins=None):

410 etree.CustomElementClassLookup.__init__(self) 411 if classes is None: 412 classes = self._default_element_classes.copy() 413 if mixins: 414 mixers = {} 415 for name, value in mixins: 416 if name == '*': 417 for n in classes.keys(): 418 mixers.setdefault(n, []).append(value) 419 else: 420 mixers.setdefault(name, []).append(value) 421 for name, mix_bases in mixers.items(): 422 cur = classes.get(name, HtmlElement) 423 bases = tuple(mix_bases + [cur]) 424 classes[name] = type(cur.__name__, bases, {}) 425 self._element_classes = classes

426

427 - def lookup(self, node_type, document, namespace, name):

428 if node_type == 'element': 429 return self._element_classes.get(name.lower(), HtmlElement) 430 elif node_type == 'comment': 431 return HtmlComment 432 elif node_type == 'PI': 433 return HtmlProcessingInstruction 434 elif node_type == 'entity': 435 return HtmlEntity 436 # Otherwise normal lookup 437 return None

438 439 ################################################################################ 440 # parsing 441 ################################################################################ 442

443 -def document_fromstring(html, **kw):

444 value = etree.HTML(html, html_parser, **kw) 445 if value is None: 446 raise etree.ParserError( 447 "Document is empty") 448 return value

449

450 -def fragments_fromstring(html, no_leading_text=False, **kw):

451 """ 452 Parses several HTML elements, returning a list of elements. 453 454 The first item in the list may be a string (though leading 455 whitespace is removed). If no_leading_text is true, then it will 456 be an error if there is leading text, and it will always be a list 457 of only elements. 458 """ 459 # FIXME: check what happens when you give html with a body, head, etc. 460 start = html[:20].lstrip().lower() 461 if not start.startswith('<html') and not start.startswith('<!doctype'): 462 html = '<html><body>%s</body></html>' % html 463 doc = document_fromstring(html, **kw) 464 assert doc.tag == 'html' 465 bodies = [e for e in doc if e.tag == 'body'] 466 assert len(bodies) == 1, ("too many bodies: %r in %r" % (bodies, html)) 467 body = bodies[0] 468 elements = [] 469 if no_leading_text and body.text and body.text.strip(): 470 raise etree.ParserError( 471 "There is leading text: %r" % body.text) 472 if body.text and body.text.strip(): 473 elements.append(body.text) 474 elements.extend(body) 475 # FIXME: removing the reference to the parent artificial document 476 # would be nice 477 return elements

478

479 -def fragment_fromstring(html, create_parent=False, **kw):

480 """ 481 Parses a single HTML element; it is an error if there is more than 482 one element, or if anything but whitespace precedes or follows the 483 element. 484 485 If create_parent is true (or is a tag name) then a parent node 486 will be created to encapsulate the HTML in a single element. 487 """ 488 if create_parent: 489 if not isinstance(create_parent, basestring): 490 create_parent = 'div' 491 return fragment_fromstring('<%s>%s</%s>' % ( 492 create_parent, html, create_parent), **kw) 493 elements = fragments_fromstring(html, no_leading_text=True) 494 if not elements: 495 raise etree.ParserError( 496 "No elements found") 497 if len(elements) > 1: 498 raise etree.ParserError( 499 "Multiple elements found (%s)" 500 % ', '.join([_element_name(e) for e in elements])) 501 el = elements[0] 502 if el.tail and el.tail.strip(): 503 raise etree.ParserError( 504 "Element followed by text: %r" % el.tail) 505 el.tail = None 506 return el

507

508 -def fromstring(html, **kw):

509 """ 510 Parse the html, returning a single element/document. 511 512 This tries to minimally parse the chunk of text, without knowing if it 513 is a fragment or a document. 514 """ 515 start = html[:10].lstrip().lower() 516 if start.startswith('<html') or start.startswith('<!doctype'): 517 # Looks like a full HTML document 518 return document_fromstring(html, **kw) 519 # otherwise, lets parse it out... 520 doc = document_fromstring(html, **kw) 521 bodies = doc.findall('body') 522 if bodies: 523 body = bodies[0] 524 if len(bodies) > 1: 525 # Somehow there are multiple bodies, which is bad, but just 526 # smash them into one body 527 for other_body in bodies[1:]: 528 if other_body.text: 529 if len(body): 530 body[-1].tail = (body[-1].tail or '') + other_body.text 531 else: 532 body.text = (body.text or '') + other_body.text 533 body.extend(other_body) 534 # We'll ignore tail 535 # I guess we are ignoring attributes too 536 other_body.drop_tree() 537 else: 538 body = None 539 heads = doc.findall('head') 540 if heads: 541 # Well, we have some sort of structure, so lets keep it all 542 head = heads[0] 543 if len(heads) > 1: 544 for other_head in heads[1:]: 545 head.extend(other_head) 546 # We don't care about text or tail in a head 547 other_head.drop_tree() 548 return doc 549 if (len(body) == 1 and (not body.text or not body.text.strip()) 550 and (not body[-1].tail or not body[-1].tail.strip())): 551 # The body has just one element, so it was probably a single 552 # element passed in 553 return body[0] 554 # Now we have a body which represents a bunch of tags which have the 555 # content that was passed in. We will create a fake container, which 556 # is the body tag, except <body> implies too much structure. 557 if _contains_block_level_tag(body): 558 body.tag = 'div' 559 else: 560 body.tag = 'span' 561 return body

562

563 -def parse(filename, parser=None, **kw):

564 """ 565 Parse a filename, URL, or file-like object into an HTML document. 566 567 You may pass the keyword argument ``base_url='http://...'`` to set 568 the base URL. 569 """ 570 if parser is None: 571 parser = html_parser 572 return etree.parse(filename, parser, **kw)

573

574 -def _contains_block_level_tag(el):

575 # FIXME: I could do this with XPath, but would that just be 576 # unnecessarily slow? 577 for el in el.getiterator(): 578 if el.tag in defs.block_tags: 579 return True 580 return False

581

582 -def _element_name(el):

583 if isinstance(el, etree.CommentBase): 584 return 'comment' 585 elif isinstance(el, basestring): 586 return 'string' 587 else: 588 return el.tag

589 590 ################################################################################ 591 # form handling 592 ################################################################################ 593

594 -class FormElement(HtmlElement):

595 """ 596 Represents a <form> element. 597 """ 598

599 - def inputs(self):

600 """ 601 Returns an accessor for all the input elements in the form. 602 603 See `InputGetter` for more information about the object. 604 """ 605 return InputGetter(self)

606 inputs = property(inputs, doc=inputs.__doc__) 607

608 - def fields__get(self):

609 """ 610 Dictionary-like object that represents all the fields in this 611 form. You can set values in this dictionary to effect the 612 form. 613 """ 614 return FieldsDict(self.inputs)

615 - def fields__set(self, value):

616 prev_keys = self.fields.keys() 617 for key, value in value.iteritems(): 618 if key in prev_keys: 619 prev_keys.remove(key) 620 self.fields[key] = value 621 for key in prev_keys: 622 if key is None: 623 # Case of an unnamed input; these aren't really 624 # expressed in form_values() anyway. 625 continue 626 self.fields[key] = None

627 628 fields = property(fields__get, fields__set, doc=fields__get.__doc__) 629

630 - def _name(self):

631 if self.get('name'): 632 return self.get('name') 633 elif self.get('id'): 634 return '#' + self.get('id') 635 return str(self.body.findall('form').index(self))

636

637 - def form_values(self):

638 """ 639 Return a list of tuples of the field values for the form. 640 This is suitable to be passed to ``urllib.urlencode()``. 641 """ 642 results = [] 643 for el in self.inputs: 644 name = el.name 645 if not name: 646 continue 647 if el.tag == 'textarea': 648 results.append((name, el.value)) 649 elif el.tag == 'select': 650 value = el.value 651 if el.multiple: 652 for v in value: 653 results.append((name, v)) 654 elif value is not None: 655 results.append((name, el.value)) 656 else: 657 assert el.tag == 'input', ( 658 "Unexpected tag: %r" % el) 659 if el.checkable and not el.checked: 660 continue 661 if el.type in ('submit', 'image', 'reset'): 662 continue 663 value = el.value 664 if value is not None: 665 results.append((name, el.value)) 666 return results

667

668 - def action__get(self):

669 """ 670 Get/set the form's ``action`` attribute. 671 """ 672 base_url = self.base_url 673 action = self.get('action') 674 if base_url and action is not None: 675 return urlparse.urljoin(base_url, action) 676 else: 677 return action

678 - def action__set(self, value):

679 self.set('action', value)

680 - def action__del(self):

681 if 'action' in self.attrib: 682 del self.attrib['action']

683 action = property(action__get, action__set, action__del, doc=action__get.__doc__) 684

685 - def method__get(self):

686 """ 687 Get/set the form's method. Always returns a capitalized 688 string, and defaults to ``'GET'`` 689 """ 690 return self.get('method', 'GET').upper()

691 - def method__set(self, value):

692 self.set('method', value.upper())

693 method = property(method__get, method__set, doc=method__get.__doc__)

694 695 HtmlElementClassLookup._default_element_classes['form'] = FormElement 696

697 -def submit_form(form, extra_values=None, open_http=None):

698 """ 699 Helper function to submit a form. Returns a file-like object, as from 700 ``urllib.urlopen()``. This object also has a ``.geturl()`` function, 701 which shows the URL if there were any redirects. 702 703 You can use this like:: 704 705 >>> form = doc.forms[0] 706 >>> form.inputs['foo'].value = 'bar' # etc 707 >>> response = form.submit() 708 >>> doc = parse(response) 709 >>> doc.make_links_absolute(response.geturl()) 710 711 To change the HTTP requester, pass a function as ``open_http`` keyword 712 argument that opens the URL for you. The function must have the following 713 signature:: 714 715 open_http(method, URL, values) 716 717 The action is one of 'GET' or 'POST', the URL is the target URL as a 718 string, and the values are a sequence of ``(name, value)`` tuples with the 719 form data. 720 """ 721 values = form.form_values() 722 if extra_values: 723 if hasattr(extra_values, 'items'): 724 extra_values = extra_values.items() 725 values.extend(extra_values) 726 if open_http is None: 727 open_http = open_http_urllib 728 return open_http(form.method, form.action, values)

729

730 -def open_http_urllib(method, url, values):

731 import urllib 732 ## FIXME: should test that it's not a relative URL or something 733 if method == 'GET': 734 if '?' in url: 735 url += '&' 736 else: 737 url += '?' 738 url += urllib.urlencode(values) 739 data = None 740 else: 741 data = urllib.urlencode(values) 742 return urllib.urlopen(url, data)

743

744 -class FieldsDict(DictMixin):

745

746 - def __init__(self, inputs):

747 self.inputs = inputs

748 - def __getitem__(self, item):

749 return self.inputs[item].value

750 - def __setitem__(self, item, value):

751 self.inputs[item].value = value

752 - def __delitem__(self, item):

753 raise KeyError( 754 "You cannot remove keys from ElementDict")

755 - def keys(self):

756 return self.inputs.keys()

757 - def __contains__(self, item):

758 return item in self.inputs

759

760 - def __repr__(self):

761 return '<%s for form %s>' % ( 762 self.__class__.__name__, 763 self.inputs.form._name())

764

765 -class InputGetter(object):

766 767 """ 768 An accessor that represents all the input fields in a form. 769 770 You can get fields by name from this, with 771 ``form.inputs['field_name']``. If there are a set of checkboxes 772 with the same name, they are returned as a list (a `CheckboxGroup` 773 which also allows value setting). Radio inputs are handled 774 similarly. 775 776 You can also iterate over this to get all input elements. This 777 won't return the same thing as if you get all the names, as 778 checkboxes and radio elements are returned individually. 779 """ 780 781 _name_xpath = etree.XPath(".//*[@name = $name and (name(.) = 'select' or name(.) = 'input' or name(.) = 'textarea')]") 782 _all_xpath = etree.XPath(".//*[name() = 'select' or name() = 'input' or name() = 'textarea']") 783

784 - def __init__(self, form):

785 self.form = form

786

787 - def __repr__(self):

788 return '<%s for form %s>' % ( 789 self.__class__.__name__, 790 self.form._name())

791 792 ## FIXME: there should be more methods, and it's unclear if this is 793 ## a dictionary-like object or list-like object 794

795 - def __getitem__(self, name):

796 results = self._name_xpath(self.form, name=name) 797 if results: 798 type = results[0].get('type') 799 if type == 'radio' and len(results) > 1: 800 group = RadioGroup(results) 801 group.name = name 802 return group 803 elif type == 'checkbox' and len(results) > 1: 804 group = CheckboxGroup(results) 805 group.name = name 806 return group 807 else: 808 # I don't like throwing away elements like this 809 return results[0] 810 else: 811 raise KeyError( 812 "No input element with the name %r" % name)

813

814 - def __contains__(self, name):

815 results = self._name_xpath(self.form, name=name) 816 return bool(results)

817

818 - def keys(self):

819 names = sets.Set() 820 for el in self: 821 names.add(el.name) 822 return list(names)

823

824 - def __iter__(self):

825 ## FIXME: kind of dumb to turn a list into an iterator, only 826 ## to have it likely turned back into a list again :( 827 return iter(self._all_xpath(self.form))

828

829 -class InputMixin(object):

830 831 """ 832 Mix-in for all input elements (input, select, and textarea) 833 """ 834 835

836 - def name__get(self):

837 """ 838 Get/set the name of the element 839 """ 840 return self.get('name')

841 - def name__set(self, value):

842 self.set('name', value)

843 - def name__del(self):

844 if 'name' in self.attrib: 845 del self.attrib['name']

846 name = property(name__get, name__set, name__del, doc=name__get.__doc__) 847

848 - def __repr__(self):

849 type = getattr(self, 'type', None) 850 if type: 851 type = ' type=%r' % type 852 else: 853 type = '' 854 return '<%s %x name=%r%s>' % ( 855 self.__class__.__name__, id(self), self.name, type)

856

857 -class TextareaElement(InputMixin, HtmlElement):

858 """ 859 ``<textarea>`` element. You can get the name with ``.name`` and 860 get/set the value with ``.value`` 861 """ 862

863 - def value__get(self):

864 """ 865 Get/set the value (which is the contents of this element) 866 """ 867 return self.text or ''

868 - def value__set(self, value):

869 self.text = value

870 - def value__del(self):

871 self.text = ''

872 value = property(value__get, value__set, value__del, doc=value__get.__doc__)

873 874 HtmlElementClassLookup._default_element_classes['textarea'] = TextareaElement 875

876 -class SelectElement(InputMixin, HtmlElement):

877 """ 878 ``<select>`` element. You can get the name with ``.name``. 879 880 ``.value`` will be the value of the selected option, unless this 881 is a multi-select element (``<select multiple>``), in which case 882 it will be a set-like object. In either case ``.value_options`` 883 gives the possible values. 884 885 The boolean attribute ``.multiple`` shows if this is a 886 multi-select. 887 """ 888

889 - def value__get(self):

890 """ 891 Get/set the value of this select (the selected option). 892 893 If this is a multi-select, this is a set-like object that 894 represents all the selected options. 895 """ 896 if self.multiple: 897 return MultipleSelectOptions(self) 898 for el in self.getiterator('option'): 899 if 'selected' in el.attrib: 900 value = el.get('value') 901 # FIXME: If value is None, what to return?, get_text()? 902 return value 903 return None

904

905 - def value__set(self, value):

906 if self.multiple: 907 if isinstance(value, basestring): 908 raise TypeError( 909 "You must pass in a sequence") 910 self.value.clear() 911 self.value.update(value) 912 return 913 if value is not None: 914 for el in self.getiterator('option'): 915 # FIXME: also if el.get('value') is None? 916 if el.get('value') == value: 917 checked_option = el 918 break 919 else: 920 raise ValueError( 921 "There is no option with the value of %r" % value) 922 for el in self.getiterator('option'): 923 if 'selected' in el.attrib: 924 del el.attrib['selected'] 925 if value is not None: 926 checked_option.set('selected', '')

927

928 - def value__del(self):

929 # FIXME: should del be allowed at all? 930 if self.multiple: 931 self.value.clear() 932 else: 933 self.value = None

934 935 value = property(value__get, value__set, value__del, doc=value__get.__doc__) 936

937 - def value_options(self):

938 """ 939 All the possible values this select can have (the ``value`` 940 attribute of all the ``<option>`` elements. 941 """ 942 return [el.get('value') for el in self.getiterator('option')]

943 value_options = property(value_options, doc=value_options.__doc__) 944

945 - def multiple__get(self):

946 """ 947 Boolean attribute: is there a ``multiple`` attribute on this element. 948 """ 949 return 'multiple' in self.attrib

950 - def multiple__set(self, value):

951 if value: 952 self.set('multiple', '') 953 elif 'multiple' in self.attrib: 954 del self.attrib['multiple']

955 multiple = property(multiple__get, multiple__set, doc=multiple__get.__doc__)

956 957 HtmlElementClassLookup._default_element_classes['select'] = SelectElement 958

959 -class MultipleSelectOptions(SetMixin):

960 """ 961 Represents all the selected options in a ``<select multiple>`` element. 962 963 You can add to this set-like option to select an option, or remove 964 to unselect the option. 965 """ 966

967 - def __init__(self, select):

968 self.select = select

969

970 - def options(self):

971 """ 972 Iterator of all the ``<option>`` elements. 973 """ 974 return self.select.getiterator('option')

975 options = property(options) 976

977 - def __iter__(self):

978 for option in self.options: 979 yield option.get('value')

980

981 - def add(self, item):

982 for option in self.options: 983 if option.get('value') == item: 984 option.set('selected', '') 985 break 986 else: 987 raise ValueError( 988 "There is no option with the value %r" % item)

989

990 - def remove(self, item):

991 for option in self.options: 992 if option.get('value') == item: 993 if 'selected' in option.attrib: 994 del option.attrib['selected'] 995 else: 996 raise ValueError( 997 "The option %r is not currently selected" % item) 998 break 999 else: 1000 raise ValueError( 1001 "There is not option with the value %r" % item)

1002

1003 - def __repr__(self):

1004 return '<%s {%s} for select name=%r>' % ( 1005 self.__class__.__name__, 1006 ', '.join([repr(v) for v in self]), 1007 self.select.name)

1008

1009 -class RadioGroup(list):

1010 """ 1011 This object represents several ``<input type=radio>`` elements 1012 that have the same name. 1013 1014 You can use this like a list, but also use the property 1015 ``.value`` to check/uncheck inputs. Also you can use 1016 ``.value_options`` to get the possible values. 1017 """ 1018

1019 - def value__get(self):

1020 """ 1021 Get/set the value, which checks the radio with that value (and 1022 unchecks any other value). 1023 """ 1024 for el in self: 1025 if 'checked' in el.attrib: 1026 return el.get('value') 1027 return None

1028

1029 - def value__set(self, value):

1030 if value is not None: 1031 for el in self: 1032 if el.get('value') == value: 1033 checked_option = el 1034 break 1035 else: 1036 raise ValueError( 1037 "There is no radio input with the value %r" % value) 1038 for el in self: 1039 if 'checked' in el.attrib: 1040 del el.attrib['checked'] 1041 if value is not None: 1042 checked_option.set('checked', '')

1043

1044 - def value__del(self):

1045 self.value = None

1046 1047 value = property(value__get, value__set, value__del, doc=value__get.__doc__) 1048

1049 - def value_options(self):

1050 """ 1051 Returns a list of all the possible values. 1052 """ 1053 return [el.get('value') for el in self]

1054 value_options = property(value_options, doc=value_options.__doc__) 1055

1056 - def __repr__(self):

1057 return '%s(%s)' % ( 1058 self.__class__.__name__, 1059 list.__repr__(self))

1060

1061 -class CheckboxGroup(list):

1062 """ 1063 Represents a group of checkboxes (``<input type=checkbox>``) that 1064 have the same name. 1065 1066 In addition to using this like a list, the ``.value`` attribute 1067 returns a set-like object that you can add to or remove from to 1068 check and uncheck checkboxes. You can also use ``.value_options`` 1069 to get the possible values. 1070 """ 1071

1072 - def value__get(self):

1073 """ 1074 Return a set-like object that can be modified to check or 1075 uncheck individual checkboxes according to their value. 1076 """ 1077 return CheckboxValues(self)

1078 - def value__set(self, value):

1079 self.value.clear() 1080 if not hasattr(value, '__iter__'): 1081 raise ValueError( 1082 "A CheckboxGroup (name=%r) must be set to a sequence (not %r)" 1083 % (self[0].name, value)) 1084 self.value.update(value)

1085 - def value__del(self):

1086 self.value.clear()

1087 value = property(value__get, value__set, value__del, doc=value__get.__doc__) 1088

1089 - def __repr__(self):

1090 return '%s(%s)' % ( 1091 self.__class__.__name__, list.__repr__(self))

1092

1093 -class CheckboxValues(SetMixin):

1094 1095 """ 1096 Represents the values of the checked checkboxes in a group of 1097 checkboxes with the same name. 1098 """ 1099

1100 - def __init__(self, group):

1101 self.group = group

1102

1103 - def __iter__(self):

1104 return iter([ 1105 el.get('value') 1106 for el in self.group 1107 if 'checked' in el.attrib])

1108

1109 - def add(self, value):

1110 for el in self.group: 1111 if el.get('value') == value: 1112 el.set('checked', '') 1113 break 1114 else: 1115 raise KeyError("No checkbox with value %r" % value)

1116

1117 - def remove(self, value):

1118 for el in self.group: 1119 if el.get('value') == value: 1120 if 'checked' in el.attrib: 1121 del el.attrib['checked'] 1122 else: 1123 raise KeyError( 1124 "The checkbox with value %r was already unchecked" % value) 1125 break 1126 else: 1127 raise KeyError( 1128 "No checkbox with value %r" % value)

1129

1130 - def __repr__(self):

1131 return '<%s {%s} for checkboxes name=%r>' % ( 1132 self.__class__.__name__, 1133 ', '.join([repr(v) for v in self]), 1134 self.group.name)

1135

1136 -class InputElement(InputMixin, HtmlElement):

1137 """ 1138 Represents an ``<input>`` element. 1139 1140 You can get the type with ``.type`` (which is lower-cased and 1141 defaults to ``'text'``). 1142 1143 Also you can get and set the value with ``.value`` 1144 1145 Checkboxes and radios have the attribute ``input.checkable == 1146 True`` (for all others it is false) and a boolean attribute 1147 ``.checked``. 1148 1149 """ 1150 1151 ## FIXME: I'm a little uncomfortable with the use of .checked

1152 - def value__get(self):

1153 """ 1154 Get/set the value of this element, using the ``value`` attribute. 1155 1156 Also, if this is a checkbox and it has no value, this defaults 1157 to ``'on'``. If it is a checkbox or radio that is not 1158 checked, this returns None. 1159 """ 1160 if self.checkable: 1161 if self.checked: 1162 return self.get('value') or 'on' 1163 else: 1164 return None 1165 return self.get('value')

1166 - def value__set(self, value):

1167 if self.checkable: 1168 if not value: 1169 self.checked = False 1170 else: 1171 self.checked = True 1172 if isinstance(value, basestring): 1173 self.set('value', value) 1174 else: 1175 self.set('value', value)

1176 - def value__del(self):

1177 if self.checkable: 1178 self.checked = False 1179 else: 1180 if 'value' in self.attrib: 1181 del self.attrib['value']

1182 value = property(value__get, value__set, value__del, doc=value__get.__doc__) 1183

1184 - def type__get(self):

1185 """ 1186 Return the type of this element (using the type attribute). 1187 """ 1188 return self.get('type', 'text').lower()

1189 - def type__set(self, value):

1190 self.set('type', value)

1191 type = property(type__get, type__set, doc=type__get.__doc__) 1192

1193 - def checkable__get(self):

1194 """ 1195 Boolean: can this element be checked? 1196 """ 1197 return self.type in ['checkbox', 'radio']

1198 checkable = property(checkable__get, doc=checkable__get.__doc__) 1199

1200 - def checked__get(self):

1201 """ 1202 Boolean attribute to get/set the presence of the ``checked`` 1203 attribute. 1204 1205 You can only use this on checkable input types. 1206 """ 1207 if not self.checkable: 1208 raise AttributeError('Not a checkable input type') 1209 return 'checked' in self.attrib

1210 - def checked__set(self, value):

1211 if not self.checkable: 1212 raise AttributeError('Not a checkable input type') 1213 if value: 1214 self.set('checked', '') 1215 else: 1216 if 'checked' in self.attrib: 1217 del self.attrib['checked']

1218 checked = property(checked__get, checked__set, doc=checked__get.__doc__)

1219 1220 HtmlElementClassLookup._default_element_classes['input'] = InputElement 1221

1222 -class LabelElement(HtmlElement):

1223 """ 1224 Represents a ``<label>`` element. 1225 1226 Label elements are linked to other elements with their ``for`` 1227 attribute. You can access this element with ``label.for_element``. 1228 """ 1229

1230 - def for_element__get(self):

1231 """ 1232 Get/set the element this label points to. Return None if it 1233 can't be found. 1234 """ 1235 id = self.get('for') 1236 if not id: 1237 return None 1238 return self.body.get_element_by_id(id)

1239 - def for_element__set(self, other):

1240 id = other.get('id') 1241 if not id: 1242 raise TypeError( 1243 "Element %r has no id attribute" % other) 1244 self.set('for', id)

1245 - def for_element__del(self):

1246 if 'id' in self.attrib: 1247 del self.attrib['id']

1248 for_element = property(for_element__get, for_element__set, for_element__del, 1249 doc=for_element__get.__doc__)

1250 1251 HtmlElementClassLookup._default_element_classes['label'] = LabelElement 1252 1253 ############################################################ 1254 ## Serialization 1255 ############################################################ 1256 1257 # This isn't a general match, but it's a match for what libxml2 1258 # specifically serialises: 1259 __replace_meta_content_type = re.compile( 1260 r'<meta http-equiv="Content-Type".*?>').sub 1261

1262 -def tostring(doc, pretty_print=False, include_meta_content_type=False):

1263 """ 1264 return HTML string representation of the document given 1265 1266 note: this will create a meta http-equiv="Content" tag in the head 1267 and may replace any that are present 1268 """ 1269 assert doc is not None 1270 html = etree.tostring(doc, method="html", pretty_print=pretty_print) 1271 if not include_meta_content_type: 1272 html = __replace_meta_content_type('', html) 1273 return html

1274

1275 -def open_in_browser(doc):

1276 """ 1277 Open the HTML document in a web browser (saving it to a temporary 1278 file to open it). 1279 """ 1280 import os 1281 import webbrowser 1282 try: 1283 write_doc = doc.write 1284 except AttributeError: 1285 write_doc = etree.ElementTree(element=doc).write 1286 fn = os.tempnam() + '.html' 1287 write_doc(fn, method="html") 1288 url = 'file://' + fn.replace(os.path.sep, '/') 1289 print url 1290 webbrowser.open(url)

1291 1292 ################################################################################ 1293 # configure Element class lookup 1294 ################################################################################ 1295

1296 -class HTMLParser(etree.HTMLParser):

1297 - def __init__(self, **kwargs):

1298 super(HTMLParser, self).__init__(**kwargs) 1299 self.setElementClassLookup(HtmlElementClassLookup())

1300

1301 -def Element(*args, **kw):

1302 v = html_parser.makeelement(*args, **kw) 1303 return v

1304 1305 html_parser = HTMLParser() 1306

Source Code for Package lxml.html