I tired to use a GenServer as a worker:
defmodule CSVServer do
use GenServer
def start_link(_opts \\ []) do
GenServer.start_link(__MODULE__, %{}, name: __MODULE__)
end
def process!(csvs_params), do: GenServer.call(__MODULE__, %{csvs_params: csvs_params})
def handle_call(%{csvs_params: csvs_params}, _from, state) do
...
end
And at my app:
def start(_type, _args) do
import Supervisor.Spec, warn: false
children = [
# Start the endpoint when the application starts
supervisor(MyApp.Endpoint, []),
# Start the Ecto repository
supervisor(MyApp.Repo, []),
worker(CSVServer, [], name: "CSVServer"),
]
The question is, do I have to explicitly name the worker? at:
worker(CSVServer, [], name: "CSVServer"),
isn't it already named at the GenServer as:
GenServer.start_link(__MODULE__, %{}, name: __MODULE__)
?
EDIT
In the generated myapp.ex I can see this:
# Here you could define other workers and supervisors as children
# worker(MyApp.SomeWorker, [], name: SomeWorker),
Related
I am trying to write a class that create methods dynamically that should executed parallel each with it's own duration with asyncio. But I am really new in the topic python asyncio and now on a point where I got stuck and have no idea how to go.
I collect servers with ip, port and command duration from config file and try to create methods in a loop and then gather these methods with async, here is my code:
import asyncio
from datetime import datetime
# from common.config import ConfigConstructor
class RCONserver:
def __init__(self, game: str, server_name=None):
self.game = game
self.server_name = server_name
# self.game_config = ConfigConstructor('cfg/rcon_server.yml')
async def send_rcon_command(self, ip: str, port: str, period: int, cnt: int):
await asyncio.sleep(int(period))
print(str(datetime.now()) + ": " + ip + " " + port)
def get_servers(self):
servers = []
for server in ['game1','game2']:
print(server)
if server[:4] == "game":
# s = self.game_config
# s.fetch_section(server)
# print(s)
servers.append(
self.send_rcon_command('192.168.178.1',
'30000',
300,
3)
return servers
async def main():
obj = RCONserver('game')
await asyncio.gather(*obj.get_servers())
asyncio.run(main())
The code is running but only one time for each server in the yml File.
What do I have to do to run it periodically for the given parameter watch period?
i think this should do the trick with loop and gather i can create functions dynamically and run each with it's own interval parallel:
import asyncio
from datetime import datetime
import random
class RCONServer:
def __init__(self):
self.rcon_loop = asyncio.get_event_loop()
def dt(self):
return datetime.now().strftime("%Y/%m/%d %H:%M:%S")
def build_rcon_functions(self):
rcon_servers = []
for server in ['game1','game2']:
rcon_servers.append(
self.rcon_command(server,
"192.168.0.1",
"30000",
"some_password",
random.randint(5, 10)
)
)
return rcon_servers
async def rcon_command(self, server: str, ip: str, port: str, passwd: str, interval: int):
while True:
await asyncio.sleep(int(interval))
print(self.dt(), ">", server)
async def run_loop(self):
rcon_tasks = self.build_rcon_functions()
try:
print(self.dt(), "> Start")
await asyncio.gather(*rcon_tasks)
self.rcon_loop.run_forever()
except KeyboardInterrupt:
pass
finally:
print(self.dt(), "> End")
self.rcon_loop.close()
obj = RCONServer()
asyncio.run(obj.run_loop())
Some suggestions for optimizing? or some hints how it can be solved better?
Is overriding from_yaml enough to register a tag from a class or is it necessary to use yaml.add_constructor(Class.yaml_tag, Class.from_yaml)? If I don't use te add_constructor method, my YAML tags are not recognized. Example of what I have:
import yaml
class Something(yaml.YAMLObject):
yaml_tag = u'!Something'
#classmethod
def from_yaml(cls,loader,node):
# Set attributes to None if not in file
values = loader.construct_mapping(node, deep=True)
attr = ['attr1','attr2']
result = {}
for val in attr:
try:
result[val] = values[val]
except KeyError:
result[val] = None
return cls(**result)
Is this enough for it to work? I'm confused with the use of from_yaml vs any other constructor you would register using the method I mentioned above. I suppose there's something fundamental I'm missing, since they say:
Subclassing YAMLObject is an easy way to define tags, constructors,
and representers for your classes. You only need to override the
yaml_tag attribute. If you want to define your custom constructor and
representer, redefine the from_yaml and to_yaml method
correspondingly.
There is indeed no need to register explicitly:
import yaml
class Something(yaml.YAMLObject):
yaml_tag = u'!Something'
def __init__(self, *args, **kw):
print('some_init', args, kw)
#classmethod
def from_yaml(cls,loader,node):
# Set attributes to None if not in file
values = loader.construct_mapping(node, deep=True)
attr = ['attr1','attr2']
result = {}
for val in attr:
try:
result[val] = values[val]
except KeyError:
result[val] = None
return cls(**result)
yaml_str = """\
test: !Something
attr1: 1
attr2: 2
"""
d = yaml.load(yaml_str)
which gives:
some_init () {'attr1': 1, 'attr2': 2}
But there is no need at all to use PyYAML's load() which is
documented to be unsafe. You can just use safe_load if you set the yaml_loader class attribute:
import yaml
class Something(yaml.YAMLObject):
yaml_tag = u'!Something'
yaml_loader = yaml.SafeLoader
def __init__(self, *args, **kw):
print('some_init', args, kw)
#classmethod
def from_yaml(cls,loader,node):
# Set attributes to None if not in file
values = loader.construct_mapping(node, deep=True)
attr = ['attr1','attr2']
result = {}
for val in attr:
try:
result[val] = values[val]
except KeyError:
result[val] = None
return cls(**result)
yaml_str = """\
test: !Something
attr1: 1
attr2: 2
"""
d = yaml.safe_load(yaml_str)
as this gives the same:
some_init () {'attr1': 1, 'attr2': 2}
(done both with Python 3.6 and Python 2.7)
The registering is done in the __init__() of the metaclass of yaml.YAMLObject:
class YAMLObjectMetaclass(type):
"""
The metaclass for YAMLObject.
"""
def __init__(cls, name, bases, kwds):
super(YAMLObjectMetaclass, cls).__init__(name, bases, kwds)
if 'yaml_tag' in kwds and kwds['yaml_tag'] is not None:
cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml)
cls.yaml_dumper.add_representer(cls, cls.to_yaml)
So maybe you are somehow interfering with that initialisation in your full class definition. Try to start with a minimal implementation as I did, and add the functionality on your class that you need until things break.
Is there a way to create a custom formatter where the passed test details with a list of except is showed?
A bit of a background for this question: we are trying to migrate to RSpec for our hardware integration and system test. The results should be pushed to CouchDB. What I am trying to achieve is a reporter that could generate a similar YAML output like the following snippet:
{
"_id": "0006b6f0-c1bd-0135-1a98-455c37fe87f1",
"_rev": "1-9c9786b4b4681ee8493f182d4fc56ef9",
"sha1_repo": "68bb327b540097c10683830f0d82acbe54a47f03",
"steps": [
{
"result": "pass",
"description": "Time for Routing expect OK: 126 micro seconds (DLC and Data also OK)"
},
{
"result": "pass",
"description": "Time for Routing expect OK: 146 micro seconds (DLC and Data also OK)"
},
{
"result": "pass",
"description": "Time for Routing expect OK: 162 micro seconds (DLC and Data also OK)"
}
],
"time_start": "1513119108000",
"time_end": "1513119108000",
"result": "pass",
"testcase_title": "Komfort_TSG_HBFS_03_to_Komfort2_TSG_HBFS_03",
"testcase_id": "TC_1zu1_BAF_Komfort_TSG_HBFS_03_to_Komfort2_TSG_HBFS_03",
"hierarchy": [
"Hardware Integration Test",
"1 - Routing",
"1.1 Normal Routing",
"1zu1_BAF_TestCases",
"CAN_to_CAN"
]
}
With failed test there is no problem to achieve this, but we need also the results from passed test in order to be able to create long term statistics.
I can override the passed event of RSPec but the example object delivers only the description and no more info.
class EliteReporter
RSpec::Core::Formatters.register self, :example_started, :example_passed, :example_failed, :example_finished
def example_passed(passed)
#output.printf "pass \n #{passed.example.description}"
end
end
Thank you in advance for any help.
Finally with the help of my colleague and thanks of the Tip from RSPec Emailing list I could do this.
I have created a Recorder class that collects the test results, than override the Expect methode. This way in the custom formatter I can collect all the passed results:
class ExpectWrapper
def initialize(_expect, _recorder, _description)
#expect = _expect
#recorder = _recorder
#description = _description
end
def to(matcher, failure_message=nil)
begin
expect_ret = #expect.to(matcher, failure_message) # test
# for tests that aggregate failures
if expect_ret.instance_of?(TrueClass)
#recorder.record(matcher.actual, matcher.description, #description)
else
#recorder.record_error(matcher.actual, matcher.description, failure_message, #description)
end
expect_ret
rescue RSpec::Expectations::ExpectationNotMetError => e
# for test that do not aggregate failures
#recorder.record_error(matcher.actual, matcher.description, failure_message, #description)
raise e
end
end
end
class Recorder
def self.start
##data = []
return Recorder.new
end
def record(expect, data, description)
##data << { :pass => true, :expect => expect, :value => data, :description => description }
self
end
def record_error(expect, data, failure_message, description)
##data << { :pass => false, :expect => expect, :value => data, :message => failure_message, :description => description }
self
end
def self.data
##data
end
def expect(object, value, description = "")
return ExpectWrapper.new(object.expect(value), self, description)
end
end
The custom formatter would look the following, is just an example, the data could be than put to JSON and pushed to Couch:
class EliteVerboseFormatter
RSpec::Core::Formatters.register self, :example_started, :example_passed, :example_failed, :example_finished
def initialize(output)
#output = output
end
def example_passed(notification)
#output.puts( format_output(notification.example, Recorder) )
end
def get_test_name( group, description)
"#{group.example.example_group}/#{description}".gsub('RSpec::ExampleGroups::','')
end
def format_output( example, recorder )
test_case = get_test_name( example.example_group, example.description)
str = "**********TEST: #{test_case} ************\n"
recorder.data.each do |d|
str += sprintf("%s: ---> expected '%-10s' to '%-20s' DESC: %s \n", d[:pass] ? 'PASS' : 'FAIL', d[:expect], d[:value], d[:description])
end
str
end
def example_failed(notification)
#output.puts(format_output( notification.example, Recorder))
exception = notification.exception
message_lines = notification.fully_formatted_lines(nil, RSpec::Core::Notifications::NullColorizer)
exception_details = if exception
{
# drop 2 removes the description (regardless of newlines) and leading blank line
:message => message_lines.drop(2).join("\n"),
:backtrace => notification.formatted_backtrace.join("\n"),
}
end
#output.puts RSpec::Core::Formatters::ConsoleCodes.wrap(exception_details[:message], :failure)
end
end
I think you can read the Module: RSpec::Core::Formatters
you might find something helpful.
P.S. I have used Cucumber for many times, and I once wanted to custom cucumber formatter to display every step's details no matter it failed or passed. I finally got the solution by reading cucumber core documents.so I think maybe rspec core document can help you to find the solution.
I find that I cannot put the code in comment, so I put it here.
edit your code as below:
class EliteReporter
RSpec::Core::Formatters.register self, :example_started, :example_passed, :example_failed, :example_finished
def example_passed(example)
example_failed(example)
end
end
I hope it can be helpful: )
I have the following code for a simple TCP server in Ruby:
# server.rb
require 'socket'
class Server
def initialize(port)
#port = port
end
def run
Socket.tcp_server_loop(#port) do |connection|
Thread.new do
loop do
puts "IO: #{IO.select([connection]).inspect} - data: #{connection.read}"
end
end
end
end
end
server = Server.new(16451)
server.run
As well as this trivial TCP client code:
# client.rb
require 'socket'
client = TCPSocket.new('localhost', 16451)
client.write('stuff')
It is my understanding that connection.read in server.rb should block if no data is present on the socket. However, when I run this on my macbook (OS X 10.12.5), it keeps spitting out the following output:
IO: [[#<Socket:fd 12>], [], []] - data: stuff
IO: [[#<Socket:fd 12>], [], []] - data:
IO: [[#<Socket:fd 12>], [], []] - data:
IO: [[#<Socket:fd 12>], [], []] - data:
IO: [[#<Socket:fd 12>], [], []] - data:
IO: [[#<Socket:fd 12>], [], []] - data:
IO: [[#<Socket:fd 12>], [], []] - data:
IO: [[#<Socket:fd 12>], [], []] - data:
...
It seems that IO.select thinks there is data available to read on the socket, while no such data has been sent.
How can I achieve a blocking read when working with sockets in Ruby? Am I overlooking something?
Matt's answer pointed me in the right direction. For future readers, here's my new code.
# server.rb
require 'socket'
class Server
BYTESIZE_OF_PACKED_INTEGER = [1].pack('i').bytesize
def initialize(port)
#port = port
end
def run
Socket.tcp_server_loop(#port) do |connection|
Thread.new do
while packed_msg_bytesize = connection.read(BYTESIZE_OF_PACKED_INTEGER)
msg_bytesize = packed_msg_bytesize.unpack('i').first
msg = connection.read(msg_bytesize)
puts msg
end
end
end
end
end
server = Server.new(16451)
server.run
And the client code.
# client.rb
require 'socket'
msg = 'stuff'
msg_bytesize = msg.bytesize
packed_msg_bytesize = [msg_bytesize].pack('i')
client = TCPSocket.new('localhost', 16451)
client.write(packed_msg_bytesize)
client.write(msg)
read will block if there is no data, but not at EOF. The IO#read docs say:
When this method is called at end of file, it returns nil or "", depending on length: read, read(nil), and read(0) return "", read(positive_integer) returns nil.
Since calling read on EOF doesn’t block, select will return the IO as readable straight away.
In your code the first call to read will block until all the data is read from the connection (i.e. the other end has closed it). From then it will be at EOF, so select will return it as ready, and read will return an empty string immediately.
I’ve just started using YAML (through pyyaml) and I was wondering if there is any way to state that the value of a key is the key name itself or the parent key.
For example
---
foo: &FOO
bar: !.
baz: !..
foo2:
<<: *FOO
…
{‘foo’: {‘bar’: ‘bar’, ‘baz’: ’foo’}, ‘foo2’:{‘bar’:’bar’, ‘baz’:’foo2’}}
(notice the dot and double dot on bar and baz respectively - those are just placeholders for getting the key name and parent key name)
I've tried using add_constructor:
def key_construct(loader, node):
# return the key here
pass
yaml.add_constructor('!.', key_construct)
but Node, doesn't hold the key (or a reference to the parent) and I couldn't find the way to get them.
EDIT:
So, here is my real use case and a solution based on Anthon's response:
I have a logger configuration file (in yaml), and I wanted to reuse some definitions there:
handlers:
base: &base_handler
(): globals.TimedRotatingFileHandlerFactory
name: ../
when: midnight
backupCount: 14
level: DEBUG
formatter: generic
syslog:
class: logging.handlers.SysLogHandler
address: ['localhost', 514]
facility: local5
level: NOTSET
formatter: syslog
access:
<<: *base_handler
error:
<<: *base_handler
loggers:
base: &base_logger
handlers: [../, syslog]
qualname: ../
access:
<<: *base_logger
error:
<<: *base_logger
handlers: [../, syslog, email]
The solution, as Anthon suggested was to traverse the configuration dictionary after is was being processed:
def expand_yaml(val, parent=None, key=None, key1=None, key2=None):
if isinstance(val, str):
if val == './':
parent[key] = key1
elif val == '../':
parent[key] = key2
elif isinstance(val, dict):
for k, v in val.items():
expand_yaml(v, val, k, k, key1)
elif isinstance(val, list):
parent[key] = val[:] # support inheritance of the list (as in *base_logger)
for index, e in enumerate(parent[key]):
expand_yaml(e, parent[key], index, key1, key2)
return val
You don't have much context when you are constructing an element, so you are not going to find your key, and certainly not the parent key, to fill in the values, without digging in the call stack for the context (the loader knows about foo, bar and baz, but not in a way you can use to determine which is the corresponding key or parent_key).
What I suggest you do is create a special node that you return with the key_construct and then after the YAML load, walk the structure that your yaml.load() returned. Unless you have other ! objects, which make it more difficult to walk the resulting combination than a pure combination of sequences/lists and mappings/dicts ¹:
import ruamel.yaml as yaml
yaml_str = """\
foo: &FOO
bar: !.
baz: !..
foo2:
<<: *FOO
"""
class Expander:
def __init__(self, tag):
self.tag = tag
def expand(self, key, parent_key):
if self.tag == '!.':
return key
elif self.tag == '!..':
return parent_key
raise NotImplementedError
def __repr__(self):
return "E({})".format(self.tag)
def expand(d, key=None, parent_key=None):
if isinstance(d, list):
for elem in d:
expand(elem, key=key, parent_key=parent_key)
elif isinstance(d, dict):
for k in d:
v = d[k]
if isinstance(v, Expander):
d[k] = v.expand(k, parent_key)
expand(d[k], key, parent_key=k)
return d
def key_construct(loader, node):
return Expander(node.tag)
yaml.add_constructor('!.', key_construct)
yaml.add_constructor('!..', key_construct)
data = yaml.load(yaml_str)
print(data)
print(expand(data))
gives you:
{'foo': {'bar': E(!.), 'baz': E(!..)}, 'foo2': {'bar': E(!.), 'baz': E(!..)}}
{'foo': {'bar': 'bar', 'baz': 'foo'}, 'foo2': {'bar': 'bar', 'baz': 'foo2'}}
¹ This was done using ruamel.yaml of which I am the author. PyYAML, of which ruamel.yaml is a functional superset, should work the same.