mirror of
https://github.com/simon987/hexlib.git
synced 2025-12-14 07:09:05 +00:00
Fix clean html (again!)
This commit is contained in:
@@ -166,7 +166,7 @@ class TestText(TestCase):
|
||||
|
||||
def test_html_no_root(self):
|
||||
text = "<a href=\"#p217709510\" class=\"quotelink\">>>217709510</a><br>Is there a<wbr>servant that is against civilization and humanity?<br>Literally instant summon."
|
||||
|
||||
|
||||
cleaned = preprocess(
|
||||
text,
|
||||
clean_html=True,
|
||||
@@ -178,9 +178,27 @@ class TestText(TestCase):
|
||||
remove_stopwords_en=False,
|
||||
remove_urls=False
|
||||
)
|
||||
|
||||
|
||||
expected = "217709510 is there a servant that is against civilization and humanity literally instant summon"
|
||||
|
||||
self.assertEqual(cleaned, expected)
|
||||
|
||||
def test_html_entity(self):
|
||||
text = "doesn't"
|
||||
|
||||
cleaned = preprocess(
|
||||
text,
|
||||
clean_html=True,
|
||||
lowercase=True,
|
||||
remove_punctuation=True,
|
||||
strip=True,
|
||||
lemmatize=False,
|
||||
fix_single_quotes=True,
|
||||
remove_stopwords_en=False,
|
||||
remove_urls=False
|
||||
)
|
||||
|
||||
expected = "doesn't"
|
||||
self.assertEqual(cleaned, expected)
|
||||
|
||||
def test_html_invalid_attribute(self):
|
||||
text = '<root><iframe width="560" height="315" src=" " title="youtube video player" frameborder="0" allowfullscreen></iframe></root>'
|
||||
|
||||
Reference in New Issue
Block a user