simon987/hexlib
1
0
Fork 0
mirror of https://github.com/simon987/hexlib.git synced 2025-04-10 06:06:41 +00:00
Code Issues Projects Releases Wiki Activity

Fix strip_quotes

Browse Source
This commit is contained in:
simon987 2021-11-16 11:48:23 -05:00
parent 3677815d57
commit 55fd4a66d2
3 changed files with 12 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hexlib/text.py
View File

@ -97,7 +97,7 @@ def preprocess(text, lowercase=False, clean_html=False, remove_punctuation=False
words = text.split() words = text.split()
if strip_quotes: if strip_quotes:
words = filter(lambda w: w.strip("\"'“"), words) words = map(lambda w: w.strip("\"'“”"), words)
if bigrams: if bigrams:
words = _transform_bigram(nltk.bigrams(chain(words, ("*",))), bigrams) words = _transform_bigram(nltk.bigrams(chain(words, ("*",))), bigrams)

2
setup.py
View File

@ -2,7 +2,7 @@ from setuptools import setup
setup( setup(
name="hexlib", name="hexlib",
version="1.68", version="1.69",
description="Misc utility methods", description="Misc utility methods",
author="simon987", author="simon987",
author_email="me@simon987.net", author_email="me@simon987.net",

10
test/test_text.py
View File

@ -247,3 +247,13 @@ class TestText(TestCase):
expected = "hello1 test1124test world" expected = "hello1 test1124test world"
self.assertEqual(" ".join(cleaned), expected) self.assertEqual(" ".join(cleaned), expected)
def test_strip_quotes(self):
text = "'hi' “test” 'hello\""
cleaned = preprocess(
text,
strip_quotes=True
)
expected = "hi test hello"
self.assertEqual(" ".join(cleaned), expected)