simon987/hexlib
1
0
Fork 0
mirror of https://github.com/simon987/hexlib.git synced 2025-04-21 18:46:42 +00:00
Code Issues Projects Releases Wiki Activity

Compare commits

base: simon987:8d8f9e87518b808bc688c691420efe2abb72a0f6
Branches Tags
simon987:master
simon987:1.89
..
compare: simon987:90d434ec7310d34d131993453167d6a385ab499b
Branches Tags
simon987:master
simon987:1.89

5 Commits
8d8f9e8751 ... 90d434ec73

Author SHA1 Message Date
simon987
90d434ec73 Add more single quotes 2021-11-16 15:32:24 -05:00
simon987
55fd4a66d2 Fix strip_quotes 2021-11-16 11:48:23 -05:00
simon987
3677815d57 Add more quotes in strip_quotes 2021-11-16 11:39:10 -05:00
simon987
1ce795a759 ... 2021-11-16 11:36:17 -05:00
simon987
e1537297d7 normalize dashes in preprocess 2021-11-16 11:34:48 -05:00
3 changed files with 18 additions and 3 deletions
Show Stats Expand all files Collapse all files

9
hexlib/text.py
View File

@ -56,9 +56,12 @@ def _transform_trigram(ngram_seq, ngrams):
yield ngram[0] yield ngram[0]
SINGLE_QUOTES = ("’", "`") SINGLE_QUOTES = ("’", "`", "‘")
SINGLE_QUOTE_TRANS = str.maketrans("".join(SINGLE_QUOTES), "".join(repeat("'", len(SINGLE_QUOTES)))) SINGLE_QUOTE_TRANS = str.maketrans("".join(SINGLE_QUOTES), "".join(repeat("'", len(SINGLE_QUOTES))))
DASHES = ("–", "⸺", "–", "—")
DASHES_TRANS = str.maketrans("".join(DASHES), "".join(repeat("-", len(DASHES))))
PUNCTUATION = ".,;:\"!?/()|*=>" PUNCTUATION = ".,;:\"!?/()|*=>"
PUNCTUATION_TRANS = str.maketrans(PUNCTUATION, " " * len(PUNCTUATION)) PUNCTUATION_TRANS = str.maketrans(PUNCTUATION, " " * len(PUNCTUATION))
@ -72,6 +75,8 @@ def preprocess(text, lowercase=False, clean_html=False, remove_punctuation=False
if fix_single_quotes: if fix_single_quotes:
text = text.translate(SINGLE_QUOTE_TRANS) text = text.translate(SINGLE_QUOTE_TRANS)
text = text.translate(DASHES_TRANS)
if remove_urls: if remove_urls:
text = LINK_RE.sub(" ", text) text = LINK_RE.sub(" ", text)
@ -92,7 +97,7 @@ def preprocess(text, lowercase=False, clean_html=False, remove_punctuation=False
words = text.split() words = text.split()
if strip_quotes: if strip_quotes:
words = filter(lambda w: w.strip("\"'"), words) words = map(lambda w: w.strip("\"'“”"), words)
if bigrams: if bigrams:
words = _transform_bigram(nltk.bigrams(chain(words, ("*",))), bigrams) words = _transform_bigram(nltk.bigrams(chain(words, ("*",))), bigrams)

2
setup.py
View File

@ -2,7 +2,7 @@ from setuptools import setup
setup( setup(
name="hexlib", name="hexlib",
version="1.66", version="1.70",
description="Misc utility methods", description="Misc utility methods",
author="simon987", author="simon987",
author_email="me@simon987.net", author_email="me@simon987.net",

10
test/test_text.py
View File

@ -247,3 +247,13 @@ class TestText(TestCase):
expected = "hello1 test1124test world" expected = "hello1 test1124test world"
self.assertEqual(" ".join(cleaned), expected) self.assertEqual(" ".join(cleaned), expected)
def test_strip_quotes(self):
text = "'hi' “test” 'hello\""
cleaned = preprocess(
text,
strip_quotes=True
)
expected = "hi test hello"
self.assertEqual(" ".join(cleaned), expected)