guppy_basecaller: env/lib/python3.7/site-packages/bs4/builder/

comparison env/lib/python3.7/site-packages/bs4/builder/_html5lib.py @ 2:6af9afd405e9 draft

"planemo upload commit 0a63dd5f4d38a1f6944587f52a8cd79874177fc1"

author	shellac
date	Thu, 14 May 2020 14:56:58 -0400
parents	26e78fe6e8c4
children

comparison

equal deleted inserted replaced

-:75ca89e9b81c
+:6af9afd405e9
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+__all__ = [
+'HTML5TreeBuilder',
+]
+import warnings
+import re
+from bs4.builder import (
+PERMISSIVE,
+HTML,
+HTML_5,
+HTMLTreeBuilder,
+)
+from bs4.element import (
+NamespacedAttribute,
+nonwhitespace_re,
+)
+import html5lib
+from html5lib.constants import (
+namespaces,
+prefixes,
+)
+from bs4.element import (
+Comment,
+Doctype,
+NavigableString,
+Tag,
+)
+try:
+# Pre-0.99999999
+from html5lib.treebuilders import _base as treebuilder_base
+new_html5lib = False
+except ImportError as e:
+# 0.99999999 and up
+from html5lib.treebuilders import base as treebuilder_base
+new_html5lib = True
+class HTML5TreeBuilder(HTMLTreeBuilder):
+"""Use html5lib to build a tree.
+Note that this TreeBuilder does not support some features common
+to HTML TreeBuilders. Some of these features could theoretically
+be implemented, but at the very least it's quite difficult,
+because html5lib moves the parse tree around as it's being built.
+* This TreeBuilder doesn't use different subclasses of NavigableString
+based on the name of the tag in which the string was found.
+* You can't use a SoupStrainer to parse only part of a document.
+"""
+NAME = "html5lib"
+features = [NAME, PERMISSIVE, HTML_5, HTML]
+# html5lib can tell us which line number and position in the
+# original file is the source of an element.
+TRACKS_LINE_NUMBERS = True
+def prepare_markup(self, markup, user_specified_encoding,
+document_declared_encoding=None, exclude_encodings=None):
+# Store the user-specified encoding for use later on.
+self.user_specified_encoding = user_specified_encoding
+# document_declared_encoding and exclude_encodings aren't used
+# ATM because the html5lib TreeBuilder doesn't use
+# UnicodeDammit.
+if exclude_encodings:
+warnings.warn("You provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.")
+yield (markup, None, None, False)
+# These methods are defined by Beautiful Soup.
+def feed(self, markup):
+if self.soup.parse_only is not None:
+warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.")
+parser = html5lib.HTMLParser(tree=self.create_treebuilder)
+self.underlying_builder.parser = parser
+extra_kwargs = dict()
+if not isinstance(markup, str):
+if new_html5lib:
+extra_kwargs['override_encoding'] = self.user_specified_encoding
+else:
+extra_kwargs['encoding'] = self.user_specified_encoding
+doc = parser.parse(markup, **extra_kwargs)
+# Set the character encoding detected by the tokenizer.
+if isinstance(markup, str):
+# We need to special-case this because html5lib sets
+# charEncoding to UTF-8 if it gets Unicode input.
+doc.original_encoding = None
+else:
+original_encoding = parser.tokenizer.stream.charEncoding[0]
+if not isinstance(original_encoding, str):
+# In 0.99999999 and up, the encoding is an html5lib
+# Encoding object. We want to use a string for compatibility
+# with other tree builders.
+original_encoding = original_encoding.name
+doc.original_encoding = original_encoding
+self.underlying_builder.parser = None
+def create_treebuilder(self, namespaceHTMLElements):
+self.underlying_builder = TreeBuilderForHtml5lib(
+namespaceHTMLElements, self.soup,
+store_line_numbers=self.store_line_numbers
+)
+return self.underlying_builder
+def test_fragment_to_document(self, fragment):
+"""See `TreeBuilder`."""
+return '<html><head></head><body>%s</body></html>' % fragment
+class TreeBuilderForHtml5lib(treebuilder_base.TreeBuilder):
+def __init__(self, namespaceHTMLElements, soup=None,
+store_line_numbers=True, **kwargs):
+if soup:
+self.soup = soup
+else:
+from bs4 import BeautifulSoup
+# TODO: Why is the parser 'html.parser' here? To avoid an
+# infinite loop?
+self.soup = BeautifulSoup(
+"", "html.parser", store_line_numbers=store_line_numbers,
+**kwargs
+)
+# TODO: What are **kwargs exactly? Should they be passed in
+# here in addition to/instead of being passed to the BeautifulSoup
+# constructor?
+super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements)
+# This will be set later to an html5lib.html5parser.HTMLParser
+# object, which we can use to track the current line number.
+self.parser = None
+self.store_line_numbers = store_line_numbers
+def documentClass(self):
+self.soup.reset()
+return Element(self.soup, self.soup, None)
+def insertDoctype(self, token):
+name = token["name"]
+publicId = token["publicId"]
+systemId = token["systemId"]
+doctype = Doctype.for_name_and_ids(name, publicId, systemId)
+self.soup.object_was_parsed(doctype)
+def elementClass(self, name, namespace):
+kwargs = {}
+if self.parser and self.store_line_numbers:
+# This represents the point immediately after the end of the
+# tag. We don't know when the tag started, but we do know
+# where it ended -- the character just before this one.
+sourceline, sourcepos = self.parser.tokenizer.stream.position()
+kwargs['sourceline'] = sourceline
+kwargs['sourcepos'] = sourcepos-1
+tag = self.soup.new_tag(name, namespace, **kwargs)
+return Element(tag, self.soup, namespace)
+def commentClass(self, data):
+return TextNode(Comment(data), self.soup)
+def fragmentClass(self):
+from bs4 import BeautifulSoup
+# TODO: Why is the parser 'html.parser' here? To avoid an
+# infinite loop?
+self.soup = BeautifulSoup("", "html.parser")
+self.soup.name = "[document_fragment]"
+return Element(self.soup, self.soup, None)
+def appendChild(self, node):
+# XXX This code is not covered by the BS4 tests.
+self.soup.append(node.element)
+def getDocument(self):
+return self.soup
+def getFragment(self):
+return treebuilder_base.TreeBuilder.getFragment(self).element
+def testSerializer(self, element):
+from bs4 import BeautifulSoup
+rv = []
+doctype_re = re.compile(r'^(.*?)(?: PUBLIC "(.*?)"(?: "(.*?)")?| SYSTEM "(.*?)")?$')
+def serializeElement(element, indent=0):
+if isinstance(element, BeautifulSoup):
+pass
+if isinstance(element, Doctype):
+m = doctype_re.match(element)
+if m:
+name = m.group(1)
+if m.lastindex > 1:
+publicId = m.group(2) or ""
+systemId = m.group(3) or m.group(4) or ""
+rv.append("""|%s<!DOCTYPE %s "%s" "%s">""" %
+(' ' * indent, name, publicId, systemId))
+else:
+rv.append("|%s<!DOCTYPE %s>" % (' ' * indent, name))
+else:
+rv.append("|%s<!DOCTYPE >" % (' ' * indent,))
+elif isinstance(element, Comment):
+rv.append("|%s<!-- %s -->" % (' ' * indent, element))
+elif isinstance(element, NavigableString):
+rv.append("|%s\"%s\"" % (' ' * indent, element))
+else:
+if element.namespace:
+name = "%s %s" % (prefixes[element.namespace],
+element.name)
+else:
+name = element.name
+rv.append("|%s<%s>" % (' ' * indent, name))
+if element.attrs:
+attributes = []
+for name, value in list(element.attrs.items()):
+if isinstance(name, NamespacedAttribute):
+name = "%s %s" % (prefixes[name.namespace], name.name)
+if isinstance(value, list):
+value = " ".join(value)
+attributes.append((name, value))
+for name, value in sorted(attributes):
+rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value))
+indent += 2
+for child in element.children:
+serializeElement(child, indent)
+serializeElement(element, 0)
+return "\n".join(rv)
+class AttrList(object):
+def __init__(self, element):
+self.element = element
+self.attrs = dict(self.element.attrs)
+def __iter__(self):
+return list(self.attrs.items()).__iter__()
+def __setitem__(self, name, value):
+# If this attribute is a multi-valued attribute for this element,
+# turn its value into a list.
+list_attr = self.element.cdata_list_attributes
+if (name in list_attr['*']
+or (self.element.name in list_attr
+and name in list_attr[self.element.name])):
+# A node that is being cloned may have already undergone
+# this procedure.
+if not isinstance(value, list):
+value = nonwhitespace_re.findall(value)
+self.element[name] = value
+def items(self):
+return list(self.attrs.items())
+def keys(self):
+return list(self.attrs.keys())
+def __len__(self):
+return len(self.attrs)
+def __getitem__(self, name):
+return self.attrs[name]
+def __contains__(self, name):
+return name in list(self.attrs.keys())
+class Element(treebuilder_base.Node):
+def __init__(self, element, soup, namespace):
+treebuilder_base.Node.__init__(self, element.name)
+self.element = element
+self.soup = soup
+self.namespace = namespace
+def appendChild(self, node):
+string_child = child = None
+if isinstance(node, str):
+# Some other piece of code decided to pass in a string
+# instead of creating a TextElement object to contain the
+# string.
+string_child = child = node
+elif isinstance(node, Tag):
+# Some other piece of code decided to pass in a Tag
+# instead of creating an Element object to contain the
+# Tag.
+child = node
+elif node.element.__class__ == NavigableString:
+string_child = child = node.element
+node.parent = self
+else:
+child = node.element
+node.parent = self
+if not isinstance(child, str) and child.parent is not None:
+node.element.extract()
+if (string_child is not None and self.element.contents
+and self.element.contents[-1].__class__ == NavigableString):
+# We are appending a string onto another string.
+# TODO This has O(n^2) performance, for input like
+# "a</a>a</a>a</a>..."
+old_element = self.element.contents[-1]
+new_element = self.soup.new_string(old_element + string_child)
+old_element.replace_with(new_element)
+self.soup._most_recent_element = new_element
+else:
+if isinstance(node, str):
+# Create a brand new NavigableString from this string.
+child = self.soup.new_string(node)
+# Tell Beautiful Soup to act as if it parsed this element
+# immediately after the parent's last descendant. (Or
+# immediately after the parent, if it has no children.)
+if self.element.contents:
+most_recent_element = self.element._last_descendant(False)
+elif self.element.next_element is not None:
+# Something from further ahead in the parse tree is
+# being inserted into this earlier element. This is
+# very annoying because it means an expensive search
+# for the last element in the tree.
+most_recent_element = self.soup._last_descendant()
+else:
+most_recent_element = self.element
+self.soup.object_was_parsed(
+child, parent=self.element,
+most_recent_element=most_recent_element)
+def getAttributes(self):
+if isinstance(self.element, Comment):
+return {}
+return AttrList(self.element)
+def setAttributes(self, attributes):
+if attributes is not None and len(attributes) > 0:
+converted_attributes = []
+for name, value in list(attributes.items()):
+if isinstance(name, tuple):
+new_name = NamespacedAttribute(*name)
+del attributes[name]
+attributes[new_name] = value
+self.soup.builder._replace_cdata_list_attribute_values(
+self.name, attributes)
+for name, value in list(attributes.items()):
+self.element[name] = value
+# The attributes may contain variables that need substitution.
+# Call set_up_substitutions manually.
+#
+# The Tag constructor called this method when the Tag was created,
+# but we just set/changed the attributes, so call it again.
+self.soup.builder.set_up_substitutions(self.element)
+attributes = property(getAttributes, setAttributes)
+def insertText(self, data, insertBefore=None):
+text = TextNode(self.soup.new_string(data), self.soup)
+if insertBefore:
+self.insertBefore(text, insertBefore)
+else:
+self.appendChild(text)
+def insertBefore(self, node, refNode):
+index = self.element.index(refNode.element)
+if (node.element.__class__ == NavigableString and self.element.contents
+and self.element.contents[index-1].__class__ == NavigableString):
+# (See comments in appendChild)
+old_node = self.element.contents[index-1]
+new_str = self.soup.new_string(old_node + node.element)
+old_node.replace_with(new_str)
+else:
+self.element.insert(index, node.element)
+node.parent = self
+def removeChild(self, node):
+node.element.extract()
+def reparentChildren(self, new_parent):
+"""Move all of this tag's children into another tag."""
+# print "MOVE", self.element.contents
+# print "FROM", self.element
+# print "TO", new_parent.element
+element = self.element
+new_parent_element = new_parent.element
+# Determine what this tag's next_element will be once all the children
+# are removed.
+final_next_element = element.next_sibling
+new_parents_last_descendant = new_parent_element._last_descendant(False, False)
+if len(new_parent_element.contents) > 0:
+# The new parent already contains children. We will be
+# appending this tag's children to the end.
+new_parents_last_child = new_parent_element.contents[-1]
+new_parents_last_descendant_next_element = new_parents_last_descendant.next_element
+else:
+# The new parent contains no children.
+new_parents_last_child = None
+new_parents_last_descendant_next_element = new_parent_element.next_element
+to_append = element.contents
+if len(to_append) > 0:
+# Set the first child's previous_element and previous_sibling
+# to elements within the new parent
+first_child = to_append[0]
+if new_parents_last_descendant is not None:
+first_child.previous_element = new_parents_last_descendant
+else:
+first_child.previous_element = new_parent_element
+first_child.previous_sibling = new_parents_last_child
+if new_parents_last_descendant is not None:
+new_parents_last_descendant.next_element = first_child
+else:
+new_parent_element.next_element = first_child
+if new_parents_last_child is not None:
+new_parents_last_child.next_sibling = first_child
+# Find the very last element being moved. It is now the
+# parent's last descendant. It has no .next_sibling and
+# its .next_element is whatever the previous last
+# descendant had.
+last_childs_last_descendant = to_append[-1]._last_descendant(False, True)
+last_childs_last_descendant.next_element = new_parents_last_descendant_next_element
+if new_parents_last_descendant_next_element is not None:
+# TODO: This code has no test coverage and I'm not sure
+# how to get html5lib to go through this path, but it's
+# just the other side of the previous line.
+new_parents_last_descendant_next_element.previous_element = last_childs_last_descendant
+last_childs_last_descendant.next_sibling = None
+for child in to_append:
+child.parent = new_parent_element
+new_parent_element.contents.append(child)
+# Now that this element has no children, change its .next_element.
+element.contents = []
+element.next_element = final_next_element
+# print "DONE WITH MOVE"
+# print "FROM", self.element
+# print "TO", new_parent_element
+def cloneNode(self):
+tag = self.soup.new_tag(self.element.name, self.namespace)
+node = Element(tag, self.soup, self.namespace)
+for key,value in self.attributes:
+node.attributes[key] = value
+return node
+def hasContent(self):
+return self.element.contents
+def getNameTuple(self):
+if self.namespace == None:
+return namespaces["html"], self.name
+else:
+return self.namespace, self.name
+nameTuple = property(getNameTuple)
+class TextNode(Element):
+def __init__(self, element, soup):
+treebuilder_base.Node.__init__(self, None)
+self.element = element
+self.soup = soup
+def cloneNode(self):
+raise NotImplementedError

Mercurial > repos > shellac > guppy_basecaller

comparison env/lib/python3.7/site-packages/bs4/builder/_html5lib.py @ 2:6af9afd405e9 draft