simon987/hexlib
1
0
Fork 0
mirror of https://github.com/simon987/hexlib.git synced 2025-04-10 06:06:41 +00:00
Code Issues Projects Releases Wiki Activity

Use imap instead of map

Browse Source
This commit is contained in:
simon987 2021-08-28 20:09:00 -04:00
parent 67c09cc10c
commit 60273fb6bd
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hexlib/text.py
View File

@ -24,11 +24,12 @@ nltk.download("wordnet", quiet=True)
lemmatizer = WordNetLemmatizer() lemmatizer = WordNetLemmatizer()
def clean_multicore(texts, processes, **kwargs): def clean_multicore(texts, processes, chunk_size=10000, **kwargs):
pool = Pool(processes=processes) pool = Pool(processes=processes)
return pool.map( yield from pool.imap(
func=partial(preprocess, **kwargs), func=partial(preprocess, **kwargs),
iterable=texts, iterable=texts,
chunksize=chunk_size
) )