simon987/hexlib
1
0
Fork 0
mirror of https://github.com/simon987/hexlib.git synced 2025-04-24 12:05:50 +00:00
Code Issues Projects Releases Wiki Activity

Compare commits

base: simon987:90d434ec7310d34d131993453167d6a385ab499b
Branches Tags
simon987:master
simon987:1.89
..
compare: simon987:8d8f9e87518b808bc688c691420efe2abb72a0f6
Branches Tags
simon987:master
simon987:1.89

No commits in common. "90d434ec7310d34d131993453167d6a385ab499b" and "8d8f9e87518b808bc688c691420efe2abb72a0f6" have entirely different histories.
90d434ec73 ... 8d8f9e8751

3 changed files with 3 additions and 18 deletions
Show Stats Expand all files Collapse all files

9
hexlib/text.py
View File

@ -56,12 +56,9 @@ def _transform_trigram(ngram_seq, ngrams):
yield ngram[0] yield ngram[0]
SINGLE_QUOTES = ("’", "`", "‘") SINGLE_QUOTES = ("’", "`")
SINGLE_QUOTE_TRANS = str.maketrans("".join(SINGLE_QUOTES), "".join(repeat("'", len(SINGLE_QUOTES)))) SINGLE_QUOTE_TRANS = str.maketrans("".join(SINGLE_QUOTES), "".join(repeat("'", len(SINGLE_QUOTES))))
DASHES = ("–", "⸺", "–", "—")
DASHES_TRANS = str.maketrans("".join(DASHES), "".join(repeat("-", len(DASHES))))
PUNCTUATION = ".,;:\"!?/()|*=>" PUNCTUATION = ".,;:\"!?/()|*=>"
PUNCTUATION_TRANS = str.maketrans(PUNCTUATION, " " * len(PUNCTUATION)) PUNCTUATION_TRANS = str.maketrans(PUNCTUATION, " " * len(PUNCTUATION))
@ -75,8 +72,6 @@ def preprocess(text, lowercase=False, clean_html=False, remove_punctuation=False
if fix_single_quotes: if fix_single_quotes:
text = text.translate(SINGLE_QUOTE_TRANS) text = text.translate(SINGLE_QUOTE_TRANS)
text = text.translate(DASHES_TRANS)
if remove_urls: if remove_urls:
text = LINK_RE.sub(" ", text) text = LINK_RE.sub(" ", text)
@ -97,7 +92,7 @@ def preprocess(text, lowercase=False, clean_html=False, remove_punctuation=False
words = text.split() words = text.split()
if strip_quotes: if strip_quotes:
words = map(lambda w: w.strip("\"'“”"), words) words = filter(lambda w: w.strip("\"'"), words)
if bigrams: if bigrams:
words = _transform_bigram(nltk.bigrams(chain(words, ("*",))), bigrams) words = _transform_bigram(nltk.bigrams(chain(words, ("*",))), bigrams)

2
setup.py
View File

@ -2,7 +2,7 @@ from setuptools import setup
setup( setup(
name="hexlib", name="hexlib",
version="1.70", version="1.66",
description="Misc utility methods", description="Misc utility methods",
author="simon987", author="simon987",
author_email="me@simon987.net", author_email="me@simon987.net",

10
test/test_text.py
View File

@ -247,13 +247,3 @@ class TestText(TestCase):
expected = "hello1 test1124test world" expected = "hello1 test1124test world"
self.assertEqual(" ".join(cleaned), expected) self.assertEqual(" ".join(cleaned), expected)
def test_strip_quotes(self):
text = "'hi' “test” 'hello\""
cleaned = preprocess(
text,
strip_quotes=True
)
expected = "hi test hello"
self.assertEqual(" ".join(cleaned), expected)