From cd5a1ac50c09c48c3b1feee20f64a4af71a89bb1 Mon Sep 17 00:00:00 2001 From: simon987 Date: Fri, 28 Jan 2022 20:11:26 -0500 Subject: [PATCH] Remove clean_multicore function --- hexlib/text.py | 9 --------- setup.py | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/hexlib/text.py b/hexlib/text.py index 1cd08cb..49aa5b8 100644 --- a/hexlib/text.py +++ b/hexlib/text.py @@ -25,15 +25,6 @@ nltk.download("wordnet", quiet=True) lemmatizer = WordNetLemmatizer() -def clean_multicore(texts, processes, chunk_size=10000, **kwargs): - pool = Pool(processes=processes) - yield from pool.imap( - func=partial(preprocess, **kwargs), - iterable=texts, - chunksize=chunk_size - ) - - def _transform_bigram(ngram_seq, ngrams): for ngram in ngram_seq: if ngram in ngrams: diff --git a/setup.py b/setup.py index 85b771f..6244ae5 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup setup( name="hexlib", - version="1.72", + version="1.73", description="Misc utility methods", author="simon987", author_email="me@simon987.net",