Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added "html5" support to use html5lib parser #238

Closed
wants to merge 2 commits into from
+18 −1
Closed

Added "html5" support to use html5lib parser #238

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
480d385
html5
aadityasinha-dotcom Mar 18, 2022
50069ac
selector for HTML5
aadityasinha-dotcom Mar 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion parsel/selector.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
from typing import Any, Dict, List, Optional, Mapping, Pattern, Union

from lxml import etree, html
from lxml.html import html5parser
from lxml.html.html5parser import document_fromstring

from .utils import flatten, iflatten, extract_regex, shorten
from .csstranslator import HTMLTranslator, GenericTranslator
Expand Down Expand Up @@ -38,6 +40,10 @@ def __init__(self, *args, **kwargs) -> None:
"_csstranslator": GenericTranslator(),
"_tostring_method": "xml",
},
'html5': {'_parser': html5parser.HTMLParser,
'_csstranslator': HTMLTranslator(),
'_tostring_method': 'html',
},
}


Expand All @@ -55,10 +61,21 @@ def create_root_node(text, parser_cls, base_url=None):
body = text.strip().replace("\x00", "").encode("utf8") or b"<html/>"
parser = parser_cls(recover=True, encoding="utf8")
root = etree.fromstring(body, parser=parser, base_url=base_url)
if parser == html5parser.HTMLParser:
try:
root = parser.parse(body, useChardet=False, override_encoding='utf8').getroot()
except ValueError:
raise TypeError('HTML5parser does not support control characters')
else:
root = etree.fromstring(body, parser=parser, base_url=base_url)
if root is None:
root = etree.fromstring(b"<html/>", parser=parser, base_url=base_url)
return root

def selector_for_html5(response):
root = document_fromstring(response.text)
selector = Selector(response, type='html', root=root)
return selector

class SelectorList(List[_SelectorType]):
"""
Expand Down Expand Up @@ -225,7 +242,7 @@ class Selector:

``text`` is a `str`` object

``type`` defines the selector type, it can be ``"html"``, ``"xml"`` or ``None`` (default).
``type`` defines the selector type, it can be ``"html"``, ``"xml"``, ``"html5"`` or ``None`` (default).
If ``type`` is ``None``, the selector defaults to ``"html"``.

``base_url`` allows setting a URL for the document. This is needed when looking up external entities with relative paths.
Expand Down