mirror of
https://github.com/simon987/hexlib.git
synced 2025-12-13 22:59:04 +00:00
Fix clean_html
This commit is contained in:
@@ -164,6 +164,25 @@ class TestText(TestCase):
|
||||
|
||||
self.assertEqual(cleaned, expected)
|
||||
|
||||
def test_html_no_root(self):
|
||||
text = "<a href=\"#p217709510\" class=\"quotelink\">>>217709510</a><br>Is there a servant that is against civilization and humanity?<br>Literally instant summon."
|
||||
|
||||
cleaned = preprocess(
|
||||
text,
|
||||
clean_html=True,
|
||||
lowercase=True,
|
||||
remove_punctuation=True,
|
||||
strip=True,
|
||||
lemmatize=False,
|
||||
fix_single_quotes=True,
|
||||
remove_stopwords_en=False,
|
||||
remove_urls=False
|
||||
)
|
||||
|
||||
expected = "217709510 is there a servant that is against civilization and humanity literally instant summon"
|
||||
|
||||
self.assertEqual(cleaned, expected)
|
||||
|
||||
def test_bigrams(self):
|
||||
text = "x A b c d e f g h"
|
||||
cleaned = preprocess(
|
||||
|
||||
Reference in New Issue
Block a user