From 3db293c867f198ccb94198618a7a66914160bbd5 Mon Sep 17 00:00:00 2001 From: simon987 Date: Sun, 14 Jun 2020 10:29:12 -0400 Subject: [PATCH] concurrency --- hexlib/concurrency.py | 47 +++++++++++++++++++++++++++++++++++++++++++ setup.py | 2 +- 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 hexlib/concurrency.py diff --git a/hexlib/concurrency.py b/hexlib/concurrency.py new file mode 100644 index 0000000..d4d851e --- /dev/null +++ b/hexlib/concurrency.py @@ -0,0 +1,47 @@ +from queue import Queue, Empty +from multiprocessing import Process +from threading import Thread + + +def queue_iter(q: Queue, **get_args): + while True: + try: + task = q.get(**get_args) + + if task is None: + break + + yield task + q.task_done() + except Empty: + break + except KeyboardInterrupt: + break + + +def queue_thread_exec(q, func, thread_count=4): + threads = [] + + for _ in range(thread_count): + t = Thread(target=func, args=(q,)) + threads.append(t) + t.start() + + q.join() + + for t in threads: + t.join() + + +def queue_process_exec(q, func, process_count=4): + processes = [] + + for _ in range(process_count): + p = Process(target=func, args=(q,)) + processes.append(p) + p.start() + + q.join() + + for p in processes: + p.join() diff --git a/setup.py b/setup.py index 8908f2b..45c071b 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup setup( name="hexlib", - version="1.9", + version="1.10", description="Misc utility methods", author="simon987", author_email="me@simon987.net",