this is my python3.8 code on MacOS, but it throws FileNotFoundError. And it's ok on python3.6 or python3.7. Very Strange!!!
# coding: utf-8
import time
from multiprocessing import Process, Lock
def task(name, lock):
lock.acquire()
print('%s 1' % name)
time.sleep(1)
print('%s 2' % name)
lock.release()
if __name__ == '__main__':
mutex = Lock()
for i in range(3):
p = Process(target=task, args=('Process %s' % i, mutex))
p.start()
And the result
......
"/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/multiprocessing/synchronize.py", line 110, in __setstate__
self._semlock = _multiprocessing.SemLock._rebuild(*state)
FileNotFoundError: [Errno 2] No such file or directory
You need to add a join in there. Note, there is no guarantee the processes will start in numerical order as you require. you would need to add a delay in before the process start to ensure it was added to the execution queue in the order you expect.
# coding: utf-8
import time
from multiprocessing import Process, Lock
def task(name, lock):
lock.acquire()
print('%s 1' % name)
time.sleep(1)
print('%s 2' % name)
lock.release()
if __name__ == '__main__':
proceses = []
mutex = Lock()
for i in range(3):
p = Process(target=task, args=('Process %s' % i, mutex))
proceses.append(p)
p.start()
for p in proceses:
p.join()
Related
i try to sequence some actions in urwid
I made a timer which run in background and communicate with the mainprocess
like this:
from multiprocessing import Process, Pipe
import time
import urwid
def show_or_exit(key):
if key in ('q', 'Q'):
raise urwid.ExitMainLoop()
class midiloop(urwid.Frame):
def __init__(self):
self.message = urwid.Text('Press Space', align='center')
self.filler = urwid.Filler(self.message, "middle")
super().__init__(urwid.Frame(self.filler))
def keypress(self, size, key):
if key == " ":
self.seq()
else:
return key
def timer(self,conn):
x = 0
while True:
if (conn.poll() == False):
pass
else:
z = conn.recv()
if (z == "kill"):
return()
conn.send(x)
x+=1
time.sleep(0.05)
def seq(self):
self.parent_conn, self.child_conn = Pipe()
self.p = Process(target=self.timer, args=(self.child_conn,))
self.p.start()
while True:
if (self.parent_conn.poll(None)):
self.y = self.parent_conn.recv()
self.message.set_text(str(self.y))
loop.draw_screen()
if ( self.y > 100 ):
self.parent_conn.send("kill")
self.message.set_text("Press Space")
return()
if __name__ == '__main__':
midiloop = midiloop()
loop = urwid.MainLoop(midiloop, unhandled_input=show_or_exit, handle_mouse=True)
loop.run()
The problem is i'm blocking urwid mainloop with while True:
So anyone can give me a solution to listen for key Q to quit the program before it reachs the end of the loop for example and more generally to interact with urwid and communicate with the subprocess
It seems to be rather complicated to combine multiprocessing and urwid.
Since you're using a timer and your class is called midiloop, I'm going to guess that maybe you want to implement a mini sequencer.
One possible way of implementing that is using an asyncio loop instead of urwid's MainLoop, and schedule events with the loop.call_later() function. I've implemented a simple drum machine with that approach in the past, using urwid for drawing the sequencer, asyncio for scheduling the play events and simpleaudio to play. You can see the code for that here: https://github.com/eliasdorneles/kickit
If you still want to implement communication with multiprocessing, I think your best bet is to use urwid.AsyncioEventLoop and the aiopipe helper for duplex communication.
It's not very minimal I'm afraid. However I did spend a day writing this Urwid frontend that starts, stops and communicates with a subprocess.
import os
import sys
from multiprocessing import Process, Pipe, Event
from collections import deque
import urwid
class suppress_stdout_stderr(object):
"""
Supresses the stdout and stderr by piping them to dev null...
The same place I send bad faith replies to my tweets
"""
def __enter__(self):
self.outnull_file = open(os.devnull, 'w')
self.errnull_file = open(os.devnull, 'w')
self.old_stdout_fileno_undup = sys.stdout.fileno()
self.old_stderr_fileno_undup = sys.stderr.fileno()
self.old_stdout_fileno = os.dup(sys.stdout.fileno())
self.old_stderr_fileno = os.dup(sys.stderr.fileno())
self.old_stdout = sys.stdout
self.old_stderr = sys.stderr
os.dup2(self.outnull_file.fileno(), self.old_stdout_fileno_undup)
os.dup2(self.errnull_file.fileno(), self.old_stderr_fileno_undup)
sys.stdout = self.outnull_file
sys.stderr = self.errnull_file
return self
def __exit__(self, *_):
sys.stdout = self.old_stdout
sys.stderr = self.old_stderr
os.dup2(self.old_stdout_fileno, self.old_stdout_fileno_undup)
os.dup2(self.old_stderr_fileno, self.old_stderr_fileno_undup)
os.close(self.old_stdout_fileno)
os.close(self.old_stderr_fileno)
self.outnull_file.close()
self.errnull_file.close()
def subprocess_main(transmit, stop_process):
with suppress_stdout_stderr():
import time
yup = ['yuuuup', 'yuuuuup', 'yeaup', 'yeoop']
nope = ['noooooooe', 'noooope', 'nope', 'nope']
mesg = 0
i = 0
while True:
i = i % len(yup)
if transmit.poll():
mesg = transmit.recv()
if mesg == 'Yup':
transmit.send(yup[i])
if mesg == 'Nope':
transmit.send(nope[i])
if stop_process.wait(0):
break
i += 1
time.sleep(2)
class SubProcess:
def __init__(self, main):
"""
Handles forking, stopping and communication with a subprocess
:param main: subprocess method to run method signature is
def main(transmit, stop_process):
transmit: is a multiprocess Pipe to send data to parent process
stop_process: is multiprocess Event to set when you want the process to exit
"""
self.main = main
self.recv, self.transmit = None, None
self.stop_process = None
self.proc = None
def fork(self):
"""
Forks and starts the subprocess
"""
self.recv, self.transmit = Pipe(duplex=True)
self.stop_process = Event()
self.proc = Process(target=self.main, args=(self.transmit, self.stop_process))
self.proc.start()
def write_pipe(self, item):
self.recv.send(item)
def read_pipe(self):
"""
Reads data sent by the process into a list and returns it
:return:
"""
item = []
if self.recv is not None:
try:
while self.recv.poll():
item += [self.recv.recv()]
except:
pass
return item
def stop(self):
"""
Sets the event to tell the process to exit.
note: this is co-operative multi-tasking, the process must respect the flag or this won't work!
"""
self.stop_process.set()
self.proc.join()
class UrwidFrontend:
def __init__(self, subprocess_main):
"""
Urwid frontend to control the subprocess and display it's output
"""
self.title = 'Urwid Frontend Demo'
self.choices = 'Start Subprocess|Quit'.split('|')
self.response = None
self.item = deque(maxlen=10)
self.event_loop = urwid.SelectEventLoop()
# start the heartbeat
self.event_loop.alarm(0, self.heartbeat)
self.main = urwid.Padding(self.main_menu(), left=2, right=2)
self.top = urwid.Overlay(self.main, urwid.SolidFill(u'\N{MEDIUM SHADE}'),
align='center', width=('relative', 60),
valign='middle', height=('relative', 60),
min_width=20, min_height=9)
self.loop = urwid.MainLoop(self.top, palette=[('reversed', 'standout', ''), ], event_loop=self.event_loop)
self.subprocess = SubProcess(subprocess_main)
def exit_program(self, button):
raise urwid.ExitMainLoop()
def main_menu(self):
body = [urwid.Text(self.title), urwid.Divider()]
for c in self.choices:
button = urwid.Button(c)
urwid.connect_signal(button, 'click', self.handle_button, c)
body.append(urwid.AttrMap(button, None, focus_map='reversed'))
return urwid.ListBox(urwid.SimpleFocusListWalker(body))
def subproc_menu(self):
self.response = urwid.Text('Waiting ...')
body = [self.response, urwid.Divider()]
choices = ['Yup', 'Nope', 'Stop Subprocess']
for c in choices:
button = urwid.Button(c)
urwid.connect_signal(button, 'click', self.handle_button, c)
body.append(urwid.AttrMap(button, None, focus_map='reversed'))
listbox = urwid.ListBox(urwid.SimpleFocusListWalker(body))
return listbox
def update_subproc_menu(self, text):
self.response.set_text(text)
def handle_button(self, button, choice):
if choice == 'Start Subprocess':
self.main.original_widget = self.subproc_menu()
self.subprocess.fork()
self.item = deque(maxlen=10)
if choice == 'Stop Subprocess':
self.subprocess.stop()
self.main.original_widget = self.main_menu()
if choice == 'Quit':
self.exit_program(button)
if choice == 'Yup':
self.subprocess.write_pipe('Yup')
if choice == 'Nope':
self.subprocess.write_pipe('Nope')
def heartbeat(self):
"""
heartbeat that runs 24 times per second
"""
# read from the process
self.item.append(self.subprocess.read_pipe())
# display it
if self.response is not None:
self.update_subproc_menu(['Subprocess started\n', f'{self.item}\n', ])
self.loop.draw_screen()
# set the next beat
self.event_loop.alarm(1 / 24, self.heartbeat)
def run(self):
self.loop.run()
if __name__ == "__main__":
app = UrwidFrontend(subprocess_main)
app.run()
I'm learning about asycio and trying to run this script I get this error:
ValueError: a coroutine was expected, got <async_generator object
mygen at 0x7fa2af959a60>
What am I missing here?
import asyncio
async def mygen(u=10):
"""Yield powers of 2."""
i = 0
while i < int(u):
yield 2 ** i
i += 1
await asyncio.sleep(0.1)
asyncio.run(mygen(5))
The asyncio.run function expects a coroutine but mygen is an asynchronous generator.
You can try something like this:
test.py:
import asyncio
async def mygen(u=10):
"""Yield powers of 2."""
i = 0
while i < int(u):
yield 2**i
i += 1
await asyncio.sleep(0.1)
async def main():
async for i in mygen(5):
print(i)
if __name__ == "__main__":
asyncio.run(main())
Test:
$ python test.py
1
2
4
8
16
References:
What does the "yield" keyword do?
PEP 525 -- Asynchronous Generators
The following code does not initiate, the pool or process method. Please help.
if __name__ == '__main__':
#client.event
async def on_message(message):
if message.guild is not None:
if (message.content.startswith('%season15')):
input = str(message.content[len('%season15'):].strip())
df = get_data(input) ##this line of code is processed
p = Pool(processes=3)
result = p.starmap(get_season_data, [(input,df,5),(input,df, 10),(input,df,15)])
processes = [Process(target=get_season_data, args=(symbol, df, i)) for i in [5,10,15]]
# Run processes
for p in processes:
p.start()
# Exit the completed processes
for p in processes:
p.join()
client.run('token#')
The code does initiate multiple python processes in task manager but does not process any of the code in get_season_data and does not throw any errors.
A client connect websocket and calls tail_log method, and new client can't connect
How to solve this problem
def on_message(self, message):
def tail_log(user,ip,port,cmd,log_path,url):
cmd = "/usr/bin/ssh -p {port} {user}#{ipaddr} {command} {logpath}" \
.format(user=user, ipaddr=ip, port=port, command=cmd, logpath=log_path)
f = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
while True:
line = f.stdout.readline().strip()
if line == '':
self.write_message('failed')
break
self.write_message(line)
tail_log(user=SSH_USER,ip=IP_ADDR,cmd=CMD,port=SSH_PORT,log_path=LOG_PATH,url=SOCKET_URL)
Your infinite loop must yield control back to Tornado's event loop, either by executing a yield, await, or by returning from the tail_log function. Since your infinite loop does not yield control to the event loop, the event loop can never process any more events, including new websocket connections.
Try using Tornado's own process module to read from your subprocess's stdout asynchronously. Something like this:
import tornado.ioloop
import tornado.process
import tornado.web
import tornado.websocket
class TailHandler(tornado.websocket.WebSocketHandler):
def open(self):
self.write_message(u"Tailing....")
self.p = tornado.process.Subprocess(
"tail -f log.log",
stdout=tornado.process.Subprocess.STREAM,
stderr=tornado.process.Subprocess.STREAM,
shell=True)
tornado.ioloop.IOLoop.current().add_callback(
lambda: self.tail(self.p.stdout))
tornado.ioloop.IOLoop.current().add_callback(
lambda: self.tail(self.p.stderr))
self.p.set_exit_callback(self.close)
async def tail(self, stream):
try:
while True:
line = await stream.read_until(b'\n')
if line:
self.write_message(line.decode('utf-8'))
else:
# "tail" exited.
return
except tornado.iostream.StreamClosedError:
# Subprocess killed.
pass
finally:
self.close()
def on_close(self):
# Client disconnected, kill the subprocess.
self.p.proc.kill()
class MainHandler(tornado.web.RequestHandler):
def get(self):
self.write("""<html><head><script>
var ws = new WebSocket("ws://localhost:8888/tail");
ws.onmessage = function (evt) {
document.write('<p>' + evt.data + '</p>');
};</script></head></html>""")
def make_app():
return tornado.web.Application([
(r"/", MainHandler),
(r"/tail", TailHandler),
])
app = make_app()
app.listen(8888)
tornado.ioloop.IOLoop.current().start()
If you're not on Python 3.5 yet, substitute #gen.coroutine for "async def", substitute "yield" for "await", and substitute "break" for "return".
I have reproduced the "Custom Completer Example" from the Qt documentation using PySide
(Python 2.7.3, PySide 1.1.2, Qt 4.8.1).
I have an issue where a win32 exception is thrown on exit (or on Mac OS X a access violation exception).
On the Mac I can see a stack trace and the issue occurs during garbage collection, where references to QObjects are apparently not consistent, such that things go bad.
I can see this crash with the following self-contained script, only if a completer insertion was accepted. I.e. type the first few letters, then accept the completion.
On the other hand, if I have seen the completion list popup, but not accepted the completion, no crash occurs on exit.
################################################################################
# Completer.py
#
# A PySide port of the Qt 4.8 "Custom Completer Example"
# http://qt-project.org/doc/qt-4.8/tools-customcompleter.html
#
################################################################################
from PySide.QtCore import *
from PySide.QtGui import *
class TextEdit(QPlainTextEdit):
def __init__(self, parent=None):
super(TextEdit, self).__init__(parent)
self.c = None
def completer(self):
return self.c
def setCompleter(self, completer):
if self.c:
QObject.disconnect(self.c, 0, self, 0)
self.c = completer
if not self.c:
return
self.c.setWidget(self)
self.c.setCompletionMode(QCompleter.PopupCompletion)
self.c.setCaseSensitivity(Qt.CaseInsensitive)
self.c.activated.connect(self.insertCompletion)
def insertCompletion(self, completion):
if self.c.widget() is not self:
return
tc = self.textCursor()
extra = len(completion) - len(self.c.completionPrefix())
tc.movePosition(QTextCursor.Left)
tc.movePosition(QTextCursor.EndOfWord)
tc.insertText(completion[-extra:])
self.setTextCursor(tc)
def textUnderCursor(self):
tc = self.textCursor()
tc.select(QTextCursor.WordUnderCursor)
return tc.selectedText()
def focusInEvent(self, event):
if self.c:
self.c.setWidget(self)
super(TextEdit, self).focusInEvent(event)
def keyPressEvent(self, e):
if self.c and self.c.popup().isVisible():
if e.key() in (Qt.Key_Enter,
Qt.Key_Return,
Qt.Key_Escape,
Qt.Key_Tab,
Qt.Key_Backtab):
e.ignore()
return
# Check for the shortcut combination Ctrl+E
isShortcut = (e.modifiers() & Qt.ControlModifier) and e.key() == Qt.Key_E
# Do not process the shortcut when we have a completion
if not self.c or not isShortcut:
super(TextEdit, self).keyPressEvent(e)
noText = not e.text()
ctrlOrShift = e.modifiers() & (Qt.ControlModifier | Qt.ShiftModifier)
if not self.c or (ctrlOrShift and noText):
return
eow = "~!##$%^&*()_+{}|:\"<>?,./;'[]\\-=" # End of word
hasModifier = (e.modifiers() != Qt.NoModifier) and not ctrlOrShift
completionPrefix = self.textUnderCursor()
if not isShortcut and \
(hasModifier or noText or len(completionPrefix) < 1 or e.text()[-1:] in eow):
self.c.popup().hide()
return
if completionPrefix != self.c.completionPrefix():
self.c.setCompletionPrefix(completionPrefix)
self.c.popup().setCurrentIndex( self.c.completionModel().index(0,0) )
cr = self.cursorRect()
cr.setWidth(self.c.popup().sizeHintForColumn(0) + \
self.c.popup().verticalScrollBar().sizeHint().width())
self.c.complete(cr)
class Completer(QMainWindow):
words = ("one",
"two",
"three",
"four")
def __init__(self, parent=None):
super(Completer, self).__init__(parent)
self.setWindowTitle("Completer")
self.textEdit = TextEdit()
self.completer = QCompleter(self)
self.completer.setModelSorting(QCompleter.CaseInsensitivelySortedModel)
self.completer.setCaseSensitivity(Qt.CaseInsensitive)
self.completer.setWrapAround(False)
self.completer.setModel(QStringListModel(Completer.words, self.completer))
self.textEdit.setCompleter(self.completer)
self.setCentralWidget(self.textEdit)
self.resize(500, 300)
self.setWindowTitle("Completer")
if __name__ == '__main__':
import sys
from PySide.QtGui import QApplication
app = QApplication(sys.argv)
window = Completer()
window.show()
sys.exit(app.exec_())