simon987/hexlib
1
0
Fork 0
mirror of https://github.com/simon987/hexlib.git synced 2025-04-18 01:16:43 +00:00
Code Issues Projects Releases Wiki Activity

Fix clean html (again)

Browse Source
This commit is contained in:
simon987 2021-08-28 19:44:10 -04:00
parent 4cff343370
commit 31b35e3a32
3 changed files with 23 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
hexlib/text.py
View File

@ -56,16 +56,11 @@ def preprocess(text, lowercase=False, clean_html=False, strip=False, remove_punc
if clean_html: if clean_html:
try: try:
text = XML_ENTITY_RE.sub(" ", text) parser = etree.XMLParser(recover=True)
text = text.replace("&", " ") root = etree.fromstring(text, parser)
text = text.replace("<br>", "<br/>")
text = "<root>" + text + "</root>"
root = etree.fromstring(text)
text = " ".join(get_text(root)) text = " ".join(get_text(root))
except Exception as e: except:
raise e
pass pass
if remove_punctuation: if remove_punctuation:

2
setup.py
View File

@ -2,7 +2,7 @@ from setuptools import setup
setup( setup(
name="hexlib", name="hexlib",
version="1.45", version="1.46",
description="Misc utility methods", description="Misc utility methods",
author="simon987", author="simon987",
author_email="me@simon987.net", author_email="me@simon987.net",

20
test/test_text.py
View File

@ -165,7 +165,7 @@ class TestText(TestCase):
self.assertEqual(cleaned, expected) self.assertEqual(cleaned, expected)
def test_html_no_root(self): def test_html_no_root(self):
text = "<a href=\"#p217709510\" class=\"quotelink\">&gt;&gt;217709510</a><br>Is there a servant that is against civilization and humanity?<br>Literally instant summon." text = "<a href=\"#p217709510\" class=\"quotelink\">&gt;&gt;217709510</a><br>Is there a<wbr>servant that is against civilization and humanity?<br>Literally instant summon."
cleaned = preprocess( cleaned = preprocess(
text, text,
@ -180,6 +180,24 @@ class TestText(TestCase):
) )
expected = "217709510 is there a servant that is against civilization and humanity literally instant summon" expected = "217709510 is there a servant that is against civilization and humanity literally instant summon"
def test_html_invalid_attribute(self):
text = '<root><iframe width="560" height="315" src=" " title="youtube video player" frameborder="0" allowfullscreen></iframe></root>'
cleaned = preprocess(
text,
clean_html=True,
lowercase=True,
remove_punctuation=True,
strip=True,
lemmatize=False,
fix_single_quotes=True,
remove_stopwords_en=False,
remove_urls=False
)
expected = ""
self.assertEqual(cleaned, expected) self.assertEqual(cleaned, expected)