hexlib/text.py

@@ -56,12 +56,9 @@ def _transform_trigram(ngram_seq, ngrams):
             yield ngram[0]
 
 
-SINGLE_QUOTES = ("’", "`", "‘")
+SINGLE_QUOTES = ("’", "`")
 SINGLE_QUOTE_TRANS = str.maketrans("".join(SINGLE_QUOTES), "".join(repeat("'", len(SINGLE_QUOTES))))
 
-DASHES = ("–", "⸺", "–", "—")
-DASHES_TRANS = str.maketrans("".join(DASHES), "".join(repeat("-", len(DASHES))))
-
 PUNCTUATION = ".,;:\"!?/()|*=>"
 PUNCTUATION_TRANS = str.maketrans(PUNCTUATION, " " * len(PUNCTUATION))
 
@@ -75,8 +72,6 @@ def preprocess(text, lowercase=False, clean_html=False, remove_punctuation=False
     if fix_single_quotes:
         text = text.translate(SINGLE_QUOTE_TRANS)
 
-    text = text.translate(DASHES_TRANS)
-
     if remove_urls:
         text = LINK_RE.sub(" ", text)
 
@@ -97,7 +92,7 @@ def preprocess(text, lowercase=False, clean_html=False, remove_punctuation=False
     words = text.split()
 
     if strip_quotes:
-        words = map(lambda w: w.strip("\"'“”"), words)
+        words = filter(lambda w: w.strip("\"'"), words)
 
     if bigrams:
         words = _transform_bigram(nltk.bigrams(chain(words, ("*",))), bigrams)

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="hexlib",
-    version="1.70",
+    version="1.66",
     description="Misc utility methods",
     author="simon987",
     author_email="me@simon987.net",

test/test_text.py

@@ -247,13 +247,3 @@ class TestText(TestCase):
         expected = "hello1 test1124test world"
 
         self.assertEqual(" ".join(cleaned), expected)
-
-    def test_strip_quotes(self):
-        text = "'hi' “test” 'hello\""
-        cleaned = preprocess(
-            text,
-            strip_quotes=True
-        )
-        expected = "hi test hello"
-
-        self.assertEqual(" ".join(cleaned), expected)