simon987/hexlib
1
0
Fork 0
mirror of https://github.com/simon987/hexlib.git synced 2025-04-17 17:16:41 +00:00
Code Issues Projects Releases Wiki Activity

normalize dashes in preprocess

Browse Source
This commit is contained in:
simon987 2021-11-16 11:34:48 -05:00
parent 8d8f9e8751
commit e1537297d7
2 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hexlib/text.py
View File

@ -59,6 +59,9 @@ def _transform_trigram(ngram_seq, ngrams):
SINGLE_QUOTES = ("’", "`") SINGLE_QUOTES = ("’", "`")
SINGLE_QUOTE_TRANS = str.maketrans("".join(SINGLE_QUOTES), "".join(repeat("'", len(SINGLE_QUOTES)))) SINGLE_QUOTE_TRANS = str.maketrans("".join(SINGLE_QUOTES), "".join(repeat("'", len(SINGLE_QUOTES))))
DASHES = ("–", "⸺", "–")
DASHES_TRANS = str.maketrans("".join(DASHES), "".join(repeat("-", len(DASHES))))
PUNCTUATION = ".,;:\"!?/()|*=>" PUNCTUATION = ".,;:\"!?/()|*=>"
PUNCTUATION_TRANS = str.maketrans(PUNCTUATION, " " * len(PUNCTUATION)) PUNCTUATION_TRANS = str.maketrans(PUNCTUATION, " " * len(PUNCTUATION))
@ -72,6 +75,8 @@ def preprocess(text, lowercase=False, clean_html=False, remove_punctuation=False
if fix_single_quotes: if fix_single_quotes:
text = text.translate(SINGLE_QUOTE_TRANS) text = text.translate(SINGLE_QUOTE_TRANS)
text = text.translate(DASHES_TRANS)
if remove_urls: if remove_urls:
text = LINK_RE.sub(" ", text) text = LINK_RE.sub(" ", text)

2
setup.py
View File

@ -2,7 +2,7 @@ from setuptools import setup
setup( setup(
name="hexlib", name="hexlib",
version="1.66", version="1.67",
description="Misc utility methods", description="Misc utility methods",
author="simon987", author="simon987",
author_email="me@simon987.net", author_email="me@simon987.net",