simon987/hexlib
1
0
Fork 0
mirror of https://github.com/simon987/hexlib.git synced 2025-04-10 06:06:41 +00:00
Code Issues Projects Releases Wiki Activity

Remove clean_multicore function

Browse Source
This commit is contained in:
simon987 2022-01-28 20:11:26 -05:00
parent 62e74ed292
commit cd5a1ac50c
2 changed files with 1 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hexlib/text.py
View File

@ -25,15 +25,6 @@ nltk.download("wordnet", quiet=True)
lemmatizer = WordNetLemmatizer() lemmatizer = WordNetLemmatizer()
def clean_multicore(texts, processes, chunk_size=10000, **kwargs):
pool = Pool(processes=processes)
yield from pool.imap(
func=partial(preprocess, **kwargs),
iterable=texts,
chunksize=chunk_size
)
def _transform_bigram(ngram_seq, ngrams): def _transform_bigram(ngram_seq, ngrams):
for ngram in ngram_seq: for ngram in ngram_seq:
if ngram in ngrams: if ngram in ngrams:

2
setup.py
View File

@ -2,7 +2,7 @@ from setuptools import setup
setup( setup(
name="hexlib", name="hexlib",
version="1.72", version="1.73",
description="Misc utility methods", description="Misc utility methods",
author="simon987", author="simon987",
author_email="me@simon987.net", author_email="me@simon987.net",