Asyncio worker class to handle parallel jobs - parallel-processing

Question

I found an example here but I dont understand how to make the code work
class Worker:
def __init__(self, func, n=3):
self.func = func
self.queue = asyncio.Queue()
self.semaphore = asyncio.Semaphore(n)
def put(self, *args):
self.queue.put_nowait(args)
async def run(self):
while True:
args = await self._get()
if args is None:
return
asyncio.ensure_future(self._target(args))
async def _get(self):
get_task = asyncio.ensure_future(self.queue.get())
join_task = asyncio.ensure_future(self.queue.join())
await asyncio.wait(coros, return_when='FIRST_COMPLETED')
if get_task.done():
return task.result()
async def _target(self, args):
try:
async with self.semaphore:
return await self.func(*args)
finally:
self.queue.task_done()
I tried then:
def work(a1,a2): print('work done',a1,a2)
W = Worker(work,n=3)
W.put(1,2)
W.put(1,2)
W.put(1,2)
result = await W.run() # for Jupyter notebooks
# asyncio.run(W.run()) # for normal python
I get error:
NameError: name 'coros' is not defined

Answer 1

I admit, the linked solution confused me, and it didn't seem to work. So, I just rewrote the Worker class which hopefully now works for you:
import asyncio
class Worker:
def __init__(self, func, n=3):
self.func = func
self.queue = asyncio.Queue()
self.semaphore = asyncio.Semaphore(n)
def put(self, *args):
self.queue.put_nowait(args)
async def run(self):
tasks = []
while True:
try:
args = self.queue.get_nowait()
except asyncio.QueueEmpty:
break
tasks.append(asyncio.ensure_future(self.do_work(args)))
await asyncio.gather(*tasks)
async def do_work(self, args):
async with self.semaphore:
await self.func(*args)
This seems like a much simpler way to do it to me. Basically, this changes Worker.run to just start a task for each item in the queue, and each task must first acquire the semaphore before calling the provided work function. And then it finishes after all the work is done.
Here is the usage:
async def work(a1, a2):
print("Starting work...", a1, a2)
await asyncio.sleep(1)
print("Finishing work...")
W = Worker(work, n=3)
W.put(1,2)
W.put(3,4)
W.put(5,6)
W.put(7,8)
W.put(9,10)
asyncio.get_event_loop().run_until_complete(W.run())
"""Output
Starting work... 1 2
Starting work... 3 4
Starting work... 5 6
Finishing work...
Finishing work...
Finishing work...
Starting work... 7 8
Starting work... 9 10
Finishing work...
Finishing work...
"""
It should be noted that you cannot use asyncio.run while also using asyncio.Semaphore this way, because asyncio.run always starts a new loop, while asyncio.Semaphore(n) sets its loop as the default loop before asyncio.run is called. This causes the semaphore to use a different loop than Worker.run.
So just using asyncio.get_event_loop.run_until_complete works fine because it uses the default loop (which the semaphore is expecting).