simon987/hexlib
1
0
Fork 0
mirror of https://github.com/simon987/hexlib.git synced 2025-04-04 02:12:59 +00:00
Code Issues Projects Releases Wiki Activity

Use imap instead of map

Browse Source
This commit is contained in:
simon987 2021-08-28 20:09:00 -04:00
parent 67c09cc10c
commit 60273fb6bd
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hexlib/text.py
View File

@ -24,11 +24,12 @@ nltk.download("wordnet", quiet=True)
lemmatizer = WordNetLemmatizer()
def clean_multicore(texts, processes, **kwargs):
def clean_multicore(texts, processes, chunk_size=10000, **kwargs):
pool = Pool(processes=processes)
return pool.map(
yield from pool.imap(
func=partial(preprocess, **kwargs),
iterable=texts,
chunksize=chunk_size
)