Related
i try to sequence some actions in urwid
I made a timer which run in background and communicate with the mainprocess
like this:
from multiprocessing import Process, Pipe
import time
import urwid
def show_or_exit(key):
if key in ('q', 'Q'):
raise urwid.ExitMainLoop()
class midiloop(urwid.Frame):
def __init__(self):
self.message = urwid.Text('Press Space', align='center')
self.filler = urwid.Filler(self.message, "middle")
super().__init__(urwid.Frame(self.filler))
def keypress(self, size, key):
if key == " ":
self.seq()
else:
return key
def timer(self,conn):
x = 0
while True:
if (conn.poll() == False):
pass
else:
z = conn.recv()
if (z == "kill"):
return()
conn.send(x)
x+=1
time.sleep(0.05)
def seq(self):
self.parent_conn, self.child_conn = Pipe()
self.p = Process(target=self.timer, args=(self.child_conn,))
self.p.start()
while True:
if (self.parent_conn.poll(None)):
self.y = self.parent_conn.recv()
self.message.set_text(str(self.y))
loop.draw_screen()
if ( self.y > 100 ):
self.parent_conn.send("kill")
self.message.set_text("Press Space")
return()
if __name__ == '__main__':
midiloop = midiloop()
loop = urwid.MainLoop(midiloop, unhandled_input=show_or_exit, handle_mouse=True)
loop.run()
The problem is i'm blocking urwid mainloop with while True:
So anyone can give me a solution to listen for key Q to quit the program before it reachs the end of the loop for example and more generally to interact with urwid and communicate with the subprocess
It seems to be rather complicated to combine multiprocessing and urwid.
Since you're using a timer and your class is called midiloop, I'm going to guess that maybe you want to implement a mini sequencer.
One possible way of implementing that is using an asyncio loop instead of urwid's MainLoop, and schedule events with the loop.call_later() function. I've implemented a simple drum machine with that approach in the past, using urwid for drawing the sequencer, asyncio for scheduling the play events and simpleaudio to play. You can see the code for that here: https://github.com/eliasdorneles/kickit
If you still want to implement communication with multiprocessing, I think your best bet is to use urwid.AsyncioEventLoop and the aiopipe helper for duplex communication.
It's not very minimal I'm afraid. However I did spend a day writing this Urwid frontend that starts, stops and communicates with a subprocess.
import os
import sys
from multiprocessing import Process, Pipe, Event
from collections import deque
import urwid
class suppress_stdout_stderr(object):
"""
Supresses the stdout and stderr by piping them to dev null...
The same place I send bad faith replies to my tweets
"""
def __enter__(self):
self.outnull_file = open(os.devnull, 'w')
self.errnull_file = open(os.devnull, 'w')
self.old_stdout_fileno_undup = sys.stdout.fileno()
self.old_stderr_fileno_undup = sys.stderr.fileno()
self.old_stdout_fileno = os.dup(sys.stdout.fileno())
self.old_stderr_fileno = os.dup(sys.stderr.fileno())
self.old_stdout = sys.stdout
self.old_stderr = sys.stderr
os.dup2(self.outnull_file.fileno(), self.old_stdout_fileno_undup)
os.dup2(self.errnull_file.fileno(), self.old_stderr_fileno_undup)
sys.stdout = self.outnull_file
sys.stderr = self.errnull_file
return self
def __exit__(self, *_):
sys.stdout = self.old_stdout
sys.stderr = self.old_stderr
os.dup2(self.old_stdout_fileno, self.old_stdout_fileno_undup)
os.dup2(self.old_stderr_fileno, self.old_stderr_fileno_undup)
os.close(self.old_stdout_fileno)
os.close(self.old_stderr_fileno)
self.outnull_file.close()
self.errnull_file.close()
def subprocess_main(transmit, stop_process):
with suppress_stdout_stderr():
import time
yup = ['yuuuup', 'yuuuuup', 'yeaup', 'yeoop']
nope = ['noooooooe', 'noooope', 'nope', 'nope']
mesg = 0
i = 0
while True:
i = i % len(yup)
if transmit.poll():
mesg = transmit.recv()
if mesg == 'Yup':
transmit.send(yup[i])
if mesg == 'Nope':
transmit.send(nope[i])
if stop_process.wait(0):
break
i += 1
time.sleep(2)
class SubProcess:
def __init__(self, main):
"""
Handles forking, stopping and communication with a subprocess
:param main: subprocess method to run method signature is
def main(transmit, stop_process):
transmit: is a multiprocess Pipe to send data to parent process
stop_process: is multiprocess Event to set when you want the process to exit
"""
self.main = main
self.recv, self.transmit = None, None
self.stop_process = None
self.proc = None
def fork(self):
"""
Forks and starts the subprocess
"""
self.recv, self.transmit = Pipe(duplex=True)
self.stop_process = Event()
self.proc = Process(target=self.main, args=(self.transmit, self.stop_process))
self.proc.start()
def write_pipe(self, item):
self.recv.send(item)
def read_pipe(self):
"""
Reads data sent by the process into a list and returns it
:return:
"""
item = []
if self.recv is not None:
try:
while self.recv.poll():
item += [self.recv.recv()]
except:
pass
return item
def stop(self):
"""
Sets the event to tell the process to exit.
note: this is co-operative multi-tasking, the process must respect the flag or this won't work!
"""
self.stop_process.set()
self.proc.join()
class UrwidFrontend:
def __init__(self, subprocess_main):
"""
Urwid frontend to control the subprocess and display it's output
"""
self.title = 'Urwid Frontend Demo'
self.choices = 'Start Subprocess|Quit'.split('|')
self.response = None
self.item = deque(maxlen=10)
self.event_loop = urwid.SelectEventLoop()
# start the heartbeat
self.event_loop.alarm(0, self.heartbeat)
self.main = urwid.Padding(self.main_menu(), left=2, right=2)
self.top = urwid.Overlay(self.main, urwid.SolidFill(u'\N{MEDIUM SHADE}'),
align='center', width=('relative', 60),
valign='middle', height=('relative', 60),
min_width=20, min_height=9)
self.loop = urwid.MainLoop(self.top, palette=[('reversed', 'standout', ''), ], event_loop=self.event_loop)
self.subprocess = SubProcess(subprocess_main)
def exit_program(self, button):
raise urwid.ExitMainLoop()
def main_menu(self):
body = [urwid.Text(self.title), urwid.Divider()]
for c in self.choices:
button = urwid.Button(c)
urwid.connect_signal(button, 'click', self.handle_button, c)
body.append(urwid.AttrMap(button, None, focus_map='reversed'))
return urwid.ListBox(urwid.SimpleFocusListWalker(body))
def subproc_menu(self):
self.response = urwid.Text('Waiting ...')
body = [self.response, urwid.Divider()]
choices = ['Yup', 'Nope', 'Stop Subprocess']
for c in choices:
button = urwid.Button(c)
urwid.connect_signal(button, 'click', self.handle_button, c)
body.append(urwid.AttrMap(button, None, focus_map='reversed'))
listbox = urwid.ListBox(urwid.SimpleFocusListWalker(body))
return listbox
def update_subproc_menu(self, text):
self.response.set_text(text)
def handle_button(self, button, choice):
if choice == 'Start Subprocess':
self.main.original_widget = self.subproc_menu()
self.subprocess.fork()
self.item = deque(maxlen=10)
if choice == 'Stop Subprocess':
self.subprocess.stop()
self.main.original_widget = self.main_menu()
if choice == 'Quit':
self.exit_program(button)
if choice == 'Yup':
self.subprocess.write_pipe('Yup')
if choice == 'Nope':
self.subprocess.write_pipe('Nope')
def heartbeat(self):
"""
heartbeat that runs 24 times per second
"""
# read from the process
self.item.append(self.subprocess.read_pipe())
# display it
if self.response is not None:
self.update_subproc_menu(['Subprocess started\n', f'{self.item}\n', ])
self.loop.draw_screen()
# set the next beat
self.event_loop.alarm(1 / 24, self.heartbeat)
def run(self):
self.loop.run()
if __name__ == "__main__":
app = UrwidFrontend(subprocess_main)
app.run()
Is overriding from_yaml enough to register a tag from a class or is it necessary to use yaml.add_constructor(Class.yaml_tag, Class.from_yaml)? If I don't use te add_constructor method, my YAML tags are not recognized. Example of what I have:
import yaml
class Something(yaml.YAMLObject):
yaml_tag = u'!Something'
#classmethod
def from_yaml(cls,loader,node):
# Set attributes to None if not in file
values = loader.construct_mapping(node, deep=True)
attr = ['attr1','attr2']
result = {}
for val in attr:
try:
result[val] = values[val]
except KeyError:
result[val] = None
return cls(**result)
Is this enough for it to work? I'm confused with the use of from_yaml vs any other constructor you would register using the method I mentioned above. I suppose there's something fundamental I'm missing, since they say:
Subclassing YAMLObject is an easy way to define tags, constructors,
and representers for your classes. You only need to override the
yaml_tag attribute. If you want to define your custom constructor and
representer, redefine the from_yaml and to_yaml method
correspondingly.
There is indeed no need to register explicitly:
import yaml
class Something(yaml.YAMLObject):
yaml_tag = u'!Something'
def __init__(self, *args, **kw):
print('some_init', args, kw)
#classmethod
def from_yaml(cls,loader,node):
# Set attributes to None if not in file
values = loader.construct_mapping(node, deep=True)
attr = ['attr1','attr2']
result = {}
for val in attr:
try:
result[val] = values[val]
except KeyError:
result[val] = None
return cls(**result)
yaml_str = """\
test: !Something
attr1: 1
attr2: 2
"""
d = yaml.load(yaml_str)
which gives:
some_init () {'attr1': 1, 'attr2': 2}
But there is no need at all to use PyYAML's load() which is
documented to be unsafe. You can just use safe_load if you set the yaml_loader class attribute:
import yaml
class Something(yaml.YAMLObject):
yaml_tag = u'!Something'
yaml_loader = yaml.SafeLoader
def __init__(self, *args, **kw):
print('some_init', args, kw)
#classmethod
def from_yaml(cls,loader,node):
# Set attributes to None if not in file
values = loader.construct_mapping(node, deep=True)
attr = ['attr1','attr2']
result = {}
for val in attr:
try:
result[val] = values[val]
except KeyError:
result[val] = None
return cls(**result)
yaml_str = """\
test: !Something
attr1: 1
attr2: 2
"""
d = yaml.safe_load(yaml_str)
as this gives the same:
some_init () {'attr1': 1, 'attr2': 2}
(done both with Python 3.6 and Python 2.7)
The registering is done in the __init__() of the metaclass of yaml.YAMLObject:
class YAMLObjectMetaclass(type):
"""
The metaclass for YAMLObject.
"""
def __init__(cls, name, bases, kwds):
super(YAMLObjectMetaclass, cls).__init__(name, bases, kwds)
if 'yaml_tag' in kwds and kwds['yaml_tag'] is not None:
cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml)
cls.yaml_dumper.add_representer(cls, cls.to_yaml)
So maybe you are somehow interfering with that initialisation in your full class definition. Try to start with a minimal implementation as I did, and add the functionality on your class that you need until things break.
I have an ABC BaseAbstract class with several getter/setter properties defined.
I want to require that the value to be set is an int and from 0 - 15.
#luminance.setter
#abstractproperty
#ValidateProperty(Exception, types=(int,), valid=lambda x: True if 0 <= x <= 15 else False)
def luminance(self, value):
"""
Set a value that indicate the level of light emitted from the block
:param value: (int): 0 (darkest) - 15 (brightest)
:return:
"""
pass
Can someone help me figure out what my ValidateProperty class/method should look like. I started with a class and called the accepts method but this is causing an error:
function object has no attribute 'func_code'
current source:
class ValidateProperty(object):
#staticmethod
def accepts(exception, *types, **kwargs):
def check_accepts(f, **kwargs):
assert len(types) == f.func_code.co_argcount
def new_f(*args, **kwds):
for i, v in enumerate(args):
if f.func_code.co_varnames[i] in types and\
not isinstance(v, types[f.func_code.co_varnames[i]]):
arg = f.func_code.co_varnames[i]
exp = types[f.func_code.co_varnames[i]]
raise exception("arg '{arg}'={r} does not match {exp}".format(arg=arg,
r=v,
exp=exp))
# del exp (unreachable)
for k,v in kwds.__iter__():
if k in types and not isinstance(v, types[k]):
raise exception("arg '{arg}'={r} does not match {exp}".format(arg=k,
r=v,
exp=types[k]))
return f(*args, **kwds)
new_f.func_name = f.func_name
return new_f
return check_accepts
One of us is confused about how decorators, descriptors (e.g. properties), and abstracts work -- I hope it's not me. ;)
Here is a rough working example:
from abc import ABCMeta, abstractproperty
class ValidateProperty:
def __init__(inst, exception, arg_type, valid):
# called on the #ValidateProperty(...) line
#
# save the exception to raise, the expected argument type, and
# the validator code for later use
inst.exception = exception
inst.arg_type = arg_type
inst.validator = valid
def __call__(inst, func):
# called after the def has finished, but before it is stored
#
# func is the def'd function, save it for later to be called
# after validating the argument
def check_accepts(self, value):
if not inst.validator(value):
raise inst.exception('value %s is not valid' % value)
func(self, value)
return check_accepts
class AbstractTestClass(metaclass=ABCMeta):
#abstractproperty
def luminance(self):
# abstract property
return
#luminance.setter
#ValidateProperty(Exception, int, lambda x: 0 <= x <= 15)
def luminance(self, value):
# abstract property with validator
return
class TestClass(AbstractTestClass):
# concrete class
val = 7
#property
def luminance(self):
# concrete property
return self.val
#luminance.setter
def luminance(self, value):
# concrete property setter
# call base class first to activate the validator
AbstractTestClass.__dict__['luminance'].__set__(self, value)
self.val = value
tc = TestClass()
print(tc.luminance)
tc.luminance = 10
print(tc.luminance)
tc.luminance = 25
print(tc.luminance)
Which results in:
7
10
Traceback (most recent call last):
File "abstract.py", line 47, in <module>
tc.luminance = 25
File "abstract.py", line 40, in luminance
AbstractTestClass.__dict__['luminance'].__set__(self, value)
File "abstract.py", line 14, in check_accepts
raise inst.exception('value %s is not valid' % value)
Exception: value 25 is not valid
A few points to think about:
The ValidateProperty is much simpler because a property setter only takes two parameters: self and the new_value
When using a class for a decorator, and the decorator takes arguments, then you will need __init__ to save the parameters, and __call__ to actually deal with the defd function
Calling a base class property setter is ugly, but you could hide that in a helper function
you might want to use a custom metaclass to ensure the validation code is run (which would also avoid the ugly base-class property call)
I suggested a metaclass above to eliminate the need for a direct call to the base class's abstractproperty, and here is an example of such:
from abc import ABCMeta, abstractproperty
class AbstractTestClassMeta(ABCMeta):
def __new__(metacls, cls, bases, clsdict):
# create new class
new_cls = super().__new__(metacls, cls, bases, clsdict)
# collect all base class dictionaries
base_dicts = [b.__dict__ for b in bases]
if not base_dicts:
return new_cls
# iterate through clsdict looking for properties
for name, obj in clsdict.items():
if not isinstance(obj, (property)):
continue
prop_set = getattr(obj, 'fset')
# found one, now look in bases for validation code
validators = []
for d in base_dicts:
b_obj = d.get(name)
if (
b_obj is not None and
isinstance(b_obj.fset, ValidateProperty)
):
validators.append(b_obj.fset)
if validators:
def check_validators(self, new_val):
for func in validators:
func(new_val)
prop_set(self, new_val)
new_prop = obj.setter(check_validators)
setattr(new_cls, name, new_prop)
return new_cls
This subclasses ABCMeta, and has ABCMeta do all of its work first, then does some additional processing. Namely:
go through the created class and look for properties
check the base classes to see if they have a matching abstractproperty
check the abstractproperty's fset code to see if it is an instance of ValidateProperty
if so, save it in a list of validators
if the list of validators is not empty
make a wrapper that will call each validator before calling the actual property's fset code
replace the found property with a new one that uses the wrapper as the setter code
ValidateProperty is a little different as well:
class ValidateProperty:
def __init__(self, exception, arg_type):
# called on the #ValidateProperty(...) line
#
# save the exception to raise and the expected argument type
self.exception = exception
self.arg_type = arg_type
self.validator = None
def __call__(self, func_or_value):
# on the first call, func_or_value is the function to use
# as the validator
if self.validator is None:
self.validator = func_or_value
return self
# every subsequent call will be to do the validation
if (
not isinstance(func_or_value, self.arg_type) or
not self.validator(None, func_or_value)
):
raise self.exception(
'%r is either not a type of %r or is outside '
'argument range' %
(func_or_value, type(func_or_value))
)
The base AbstractTestClass now uses the new AbstractTestClassMeta, and has the validator code directly in the abstractproperty:
class AbstractTestClass(metaclass=AbstractTestClassMeta):
#abstractproperty
def luminance(self):
# abstract property
pass
#luminance.setter
#ValidateProperty(Exception, int)
def luminance(self, value):
# abstract property validator
return 0 <= value <= 15
The final class is the same:
class TestClass(AbstractTestClass):
# concrete class
val = 7
#property
def luminance(self):
# concrete property
return self.val
#luminance.setter
def luminance(self, value):
# concrete property setter
# call base class first to activate the validator
# AbstractTestClass.__dict__['luminance'].__set__(self, value)
self.val = value
Understand from many articles on stack overflow that the filter method in the tweepy.streaming.stream class uses a logical OR for track and location arguements
so the below will return either tweets from location=USA or with a word ""
streamObj = tweepy.streaming.Stream(oauthObject
,EchoStreamListener(api=apiInstance,
dump_json=args.json,
numtweets=args.numtweets))
keyWordList = ['panthers','falcon']
GEOBOX_USA = [-125,25.1,-60.5,49.1]
streamObj.filter(locations=GEOBOX_USA, track=keyWordList, languages=['en'])
This solution (How to add a location filter to tweepy module
) to check keywords in the on_status method works great, but if i needed to store the entire json variable i think i would have to use the on_data
so changed the on_data (as shown in code below), but get an error:
File "/Library/Python/2.7/site-packages/tweepy/streaming.py", line 294, in _run
raise exception
KeyError: 'text'
-- coding: utf-8 --
from types import *
import tweepy
import json
import argparse
import io
class EchoStreamListener(tweepy.StreamListener):
def __init__(self, api, dump_json=False, numtweets=0):
self.api = api
self.dump_json = dump_json
self.count = 0
self.limit = int(numtweets)
super(tweepy.StreamListener, self).__init__()
# def on_status(self, status):
# if any(keyWord in status.text.lower() for keyWord in keyWordList):
# print status.text
#
# self.count+=1
# return False if self.count == self.limit else True
# else:
# return True # Don't kill the stream
def on_data(self, tweet):
tweet_data = json.loads(tweet) # This allows the JSON data be used as a normal dictionary:
if any(keyWord in tweet_data['text'] for keyWord in keyWordList):
if self.dump_json:
print json.dumps(tweet_data)
saveFile.write(unicode(tweet) + "\n")
self.count+=1
return False if self.count == self.limit else True
else:
print tweet_data['created_at','name','text'].encode("utf-8").rstrip()
def on_error(self, status_code):
print >> sys.stderr, 'Encountered error with status code:', status_code
return True
def get_parser():
parser = argparse.ArgumentParser(add_help=True)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument(
'-j', '--json',
action='store_true',
help='dump each tweet as a json string'
)
group.add_argument(
'-t', '--text',
dest='json',
action='store_false',
help='dump each tweet\'s text'
)
parser.add_argument(
'-n', '--numtweets',
metavar='numtweets',
help='set number of tweets to retrieve'
)
return parser
if __name__ == '__main__':
oauthObject = tweepy.OAuthHandler(myconsumer_key, myconsumer_secret)
oauthObject.set_access_token(myaccess_key,myaccess_secret)
apiInstance = tweepy.API(oauthObject)
parser = get_parser()
args = parser.parse_args()
streamObj = tweepy.streaming.Stream(oauthObject
,EchoStreamListener(api=apiInstance,
dump_json=args.json,
numtweets=args.numtweets))
keyWordList = ['panthers','falcon']
GEOBOX_USA = [-125,25.1,-60.5,49.1]
saveFile = io.open('/Users/deepaktanna/raw_tweets.json', 'w', encoding='utf-8')
streamObj.filter(locations=GEOBOX_USA, languages=['en'])
saveFile.close()
this is my form:
class IPTrackerSearchForm(forms.Form):
keyword = forms.CharField(max_length=100, widget=forms.TextInput(attrs={'size':'50'}))
search_in = forms.ChoiceField(required=False, choices=ANY_CHOICE + MODULE_SEARCH_IN_CHOICES)
product = forms.CharField(max_length=64,widget=forms.TextInput(attrs={'size':'50'}))
family = forms.CharField(max_length=64,widget=forms.TextInput(attrs={'size':'50'}))
temp_result = Merlin.objects.values('build').distinct()
result = [(value['build'], value['build']) for value in temp_result]
build = forms.ChoiceField(choices=ANY_CHOICE + result)
circuit_name = forms.CharField(max_length=256,widget=forms.TextInput(attrs={'size':'50'}))
parameterization = forms.CharField(max_length=1024,widget=forms.TextInput(attrs={'size':'50'}))
metric = forms.CharField(max_length=64,widget=forms.TextInput(attrs={'size':'50'}))
show_in_one_page = forms.BooleanField(required=False, label="Show filtered result in one page", widget=forms.CheckboxInput(attrs={'class':'checkbox'}))
def __init__(self, *args, **kwargs):
super(IPTrackerSearchForm, self).__init__(*args, **kwargs)
temp_result = Merlin.objects.values('build').distinct()
self.result = [(value['build'], value['build']) for value in temp_result]
self.build = forms.ChoiceField(choices=ANY_CHOICE + self.result)
print self.result
With the purpose that, each time I refresh the webpage, when have new record to "build" column in database. It should update to the drop down box "build" here but It never update unless restart the server. I use print and see that __init__ detect new record but can not refect to build in Class.
Many thanks
You actually need to update self.fields['build'] instead of self.build.
def __init__(self, *args, **kwargs):
super(IPTrackerSearchForm, self).__init__(*args, **kwargs)
temp_result = Merlin.objects.values('build').distinct()
result = [(value['build'], value['build']) for value in temp_result]
self.fields['build'] = forms.ChoiceField(choices=ANY_CHOICE + result)
print result
Because you're not updating self.fields, you are seeing the result of the query at compile time, not execution.