simon987/hexlib
1
0
Fork 0
mirror of https://github.com/simon987/hexlib.git synced 2025-04-18 01:16:43 +00:00
Code Issues Projects Releases Wiki Activity

Fix clean html (again!)

Browse Source
This commit is contained in:
simon987 2021-08-28 19:59:04 -04:00
parent 31b35e3a32
commit a7bf5b2d15
4 changed files with 25 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hexlib/regex.py
View File

@ -3,5 +3,5 @@ import re
LINK_RE = re.compile(r"(https?://[\w\-_.]+\.[a-z]{2,4}([^\s<'\"]*|$))") LINK_RE = re.compile(r"(https?://[\w\-_.]+\.[a-z]{2,4}([^\s<'\"]*|$))")
HTML_HREF_RE = re.compile(r"href=\"([^\"]+)\"") HTML_HREF_RE = re.compile(r"href=\"([^\"]+)\"")
WHITESPACE_RE = re.compile(r"\s+") WHITESPACE_RE = re.compile(r"\s+")
PUNCTUATION_RE = re.compile(r"[.,;:\"!?/()|*=]+") PUNCTUATION_RE = re.compile(r"[.,;:\"!?/()|*=>]+")
XML_ENTITY_RE = re.compile(r"&[a-z]+;") XML_ENTITY_RE = re.compile(r"&[a-z]+;")

2
hexlib/text.py
View File

@ -56,6 +56,8 @@ def preprocess(text, lowercase=False, clean_html=False, strip=False, remove_punc
if clean_html: if clean_html:
try: try:
text = "<root>" + text + "</root>"
parser = etree.XMLParser(recover=True) parser = etree.XMLParser(recover=True)
root = etree.fromstring(text, parser) root = etree.fromstring(text, parser)

2
setup.py
View File

@ -2,7 +2,7 @@ from setuptools import setup
setup( setup(
name="hexlib", name="hexlib",
version="1.46", version="1.47",
description="Misc utility methods", description="Misc utility methods",
author="simon987", author="simon987",
author_email="me@simon987.net", author_email="me@simon987.net",

24
test/test_text.py
View File

@ -166,7 +166,7 @@ class TestText(TestCase):
def test_html_no_root(self): def test_html_no_root(self):
text = "<a href=\"#p217709510\" class=\"quotelink\">&gt;&gt;217709510</a><br>Is there a<wbr>servant that is against civilization and humanity?<br>Literally instant summon." text = "<a href=\"#p217709510\" class=\"quotelink\">&gt;&gt;217709510</a><br>Is there a<wbr>servant that is against civilization and humanity?<br>Literally instant summon."
cleaned = preprocess( cleaned = preprocess(
text, text,
clean_html=True, clean_html=True,
@ -178,9 +178,27 @@ class TestText(TestCase):
remove_stopwords_en=False, remove_stopwords_en=False,
remove_urls=False remove_urls=False
) )
expected = "217709510 is there a servant that is against civilization and humanity literally instant summon" expected = "217709510 is there a servant that is against civilization and humanity literally instant summon"
self.assertEqual(cleaned, expected)
def test_html_entity(self):
text = "doesn&#039;t"
cleaned = preprocess(
text,
clean_html=True,
lowercase=True,
remove_punctuation=True,
strip=True,
lemmatize=False,
fix_single_quotes=True,
remove_stopwords_en=False,
remove_urls=False
)
expected = "doesn't"
self.assertEqual(cleaned, expected)
def test_html_invalid_attribute(self): def test_html_invalid_attribute(self):
text = '<root><iframe width="560" height="315" src=" " title="youtube video player" frameborder="0" allowfullscreen></iframe></root>' text = '<root><iframe width="560" height="315" src=" " title="youtube video player" frameborder="0" allowfullscreen></iframe></root>'