simon987/hexlib
1
0
Fork 0
mirror of https://github.com/simon987/hexlib.git synced 2025-04-18 01:16:43 +00:00
Code Issues Projects Releases Wiki Activity

Fix clean_html

Browse Source
This commit is contained in:
simon987 2021-08-28 19:33:11 -04:00
parent db3e191983
commit 4d6c8018df
3 changed files with 31 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hexlib/regex.py
View File

@ -4,3 +4,4 @@ LINK_RE = re.compile(r"(https?://[\w\-_.]+\.[a-z]{2,4}([^\s<'\"]*|$))")
HTML_HREF_RE = re.compile(r"href=\"([^\"]+)\"") HTML_HREF_RE = re.compile(r"href=\"([^\"]+)\"")
WHITESPACE_RE = re.compile(r"\s+") WHITESPACE_RE = re.compile(r"\s+")
PUNCTUATION_RE = re.compile(r"[.,;:\"!?/()|*=]+") PUNCTUATION_RE = re.compile(r"[.,;:\"!?/()|*=]+")
XML_ENTITY_RE = re.compile(r"&[a-z]+;")

15
hexlib/text.py
View File

@ -6,7 +6,7 @@ from lxml import etree
from nltk.corpus import stopwords from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer from nltk.stem import WordNetLemmatizer
from .regex import WHITESPACE_RE, PUNCTUATION_RE, LINK_RE from .regex import WHITESPACE_RE, PUNCTUATION_RE, LINK_RE, XML_ENTITY_RE
get_text = etree.XPath("//text()") get_text = etree.XPath("//text()")
@ -56,9 +56,16 @@ def preprocess(text, lowercase=False, clean_html=False, strip=False, remove_punc
if clean_html: if clean_html:
try: try:
root = etree.fromstring(text.replace("&", "")) text = XML_ENTITY_RE.sub(" ", text)
text = "".join(get_text(root)) text = text.replace("&", " ")
except: text = text.replace("<br>", "<br/>")
text = "<root>" + text + "</root>"
root = etree.fromstring(text)
text = " ".join(get_text(root))
except Exception as e:
raise e
pass pass
if remove_punctuation: if remove_punctuation:

19
test/test_text.py
View File

@ -164,6 +164,25 @@ class TestText(TestCase):
self.assertEqual(cleaned, expected) self.assertEqual(cleaned, expected)
def test_html_no_root(self):
text = "<a href=\"#p217709510\" class=\"quotelink\">&gt;&gt;217709510</a><br>Is there a servant that is against civilization and humanity?<br>Literally instant summon."
cleaned = preprocess(
text,
clean_html=True,
lowercase=True,
remove_punctuation=True,
strip=True,
lemmatize=False,
fix_single_quotes=True,
remove_stopwords_en=False,
remove_urls=False
)
expected = "217709510 is there a servant that is against civilization and humanity literally instant summon"
self.assertEqual(cleaned, expected)
def test_bigrams(self): def test_bigrams(self):
text = "x A b c d e f g h" text = "x A b c d e f g h"
cleaned = preprocess( cleaned = preprocess(