From 60273fb6bdcb30bae96df91cb6e7f07623cd78d0 Mon Sep 17 00:00:00 2001
From: simon987 <me@simon987.net>
Date: Sat, 28 Aug 2021 20:09:00 -0400
Subject: [PATCH] Use imap instead of map

---
 hexlib/text.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/hexlib/text.py b/hexlib/text.py
index 7bbcbc5..d4916c0 100644
--- a/hexlib/text.py
+++ b/hexlib/text.py
@@ -24,11 +24,12 @@ nltk.download("wordnet", quiet=True)
 lemmatizer = WordNetLemmatizer()
 
 
-def clean_multicore(texts, processes, **kwargs):
+def clean_multicore(texts, processes, chunk_size=10000, **kwargs):
     pool = Pool(processes=processes)
-    return pool.map(
+    yield from pool.imap(
         func=partial(preprocess, **kwargs),
         iterable=texts,
+        chunksize=chunk_size
     )