Asychronous server hanging in python - python-asyncio

Am currently using python asynchronous server to server some clients , the server works well at first then eventually hangs without showing if it is receiving and requests from clients, when I press ctr-c it then shows that it receives the requests from the clients and it does not stop, it just continues to revieve requests .
I don't where this bug is emanating from.
Thank u in advance
import asyncio, json
from collections import Coroutine
from typing import Any
import Engine
import struct
header_struct = struct.Struct("!Q") # messages up to 2**64 - 1 in length
async def recvall(reader, length):
blocks = []
while length:
block = await reader.read(length)
if not block:
raise EOFError('socket closed with {} bytes left in this block'.format(length))
length -= len(block)
blocks.append(block)
return b''.join(blocks)
async def get_block(reader):
data = await recvall(reader, header_struct.size)
(block_length,) = header_struct.unpack(data)
return await recvall(reader, block_length)
async def put_block(writer, message):
block_length = len(message)
writer.write(header_struct.pack(block_length))
writer.write(message)
# await writer.drain()
async def handle_conversation(reader, writer):
address__ = writer.get_extra_info("peername")
print("Accepted connection from {}".format(address__))
while True:
# ************************try to check if there data to send*********************************
try:
block = await get_block(reader)
# decode the data
data = block.decode()
decoded_data = json.loads(data)
# dont forget to make this synchronous
answer = await Engine.get_answer(decoded_data["Task"], decoded_data["content"])
# don't forget to check in there is necessary data to push and making sure data is conveyed
await put_block(writer, answer)
print(answer)
except Exception as e:
raise
if __name__ == '__main__':
address = Engine.parse_command_line("asyncio server using coroutine")
# loop = asyncio.get_event_loop()
# coro = asyncio.start_server(handle_conversation, *address)
async def main():
server = await asyncio.start_server(
handle_conversation, *address)
addr = server.sockets[0].getsockname()
print(f'Serving on {addr}')
async with server:
await server.serve_forever()
asyncio.run(main(), debug=True)
Engine Code
import argparse
import json
import time
import upload_pic_and_search
import accept_connections
import connect
import opinion_poll
import share
import zmq
from jsonrpclib import Server
context = zmq.Context()
aphorisms = {"share": share.share_,
"poll": opinion_poll.add_poll,
"add_profile_pic": upload_pic_and_search.profile_pic,
"connect": connect.connect,
"accept_connection": accept_connections.accept_connection}
def sighn_up(doc):
"""this function will connect to sighn up """
proxy = Server('http://localhost:7002')
answer = proxy.sighn_up(doc)
return answer
def Verification(doc):
"""Code verification routine"""
proxy = Server('http://localhost:7002')
answer = proxy.verify(doc)
return answer
def login(doc):
"""This function handkes authetication"""
proxy = Server('http://localhost:7002')
answer = proxy.autheticate(doc)
return answer
def post(doc):
"""connect to server that handles posts"""
proxy = Server('http://localhost:6700')
answer = proxy.post(doc)
return answer
def comment(doc):
"""connect to the server that stores comments"""
proxy = Server('http://localhost:6701')
answer = proxy.comments_(doc)
return answer
def reply(doc):
"""store the reply"""
proxy = Server('http://localhost:6702')
answer = proxy.reply(doc)
return answer
def share(doc):
"""share the post"""
proxy = Server('http://localhost:6703')
answer = proxy.share(doc)
return answer
def likes(doc):
"""connect to the likes queue"""
zcontext = zmq.Context()
osock = zcontext.socket(zmq.PUSH)
osock.connect("tcp://127.0.0.1:6704")
osock.send_json(doc)
return {"Task": "like", "like": True}
def follow(doc):
"""handles the follow coroutine"""
zcontext = zmq.Context()
osock = zcontext.socket(zmq.PUSH)
osock.connect("tcp://127.0.0.1:6705")
osock.send_json(doc)
def connect(doc):
"""connect to routine for connection"""
zcontext = zmq.Context()
osock = zcontext.socket(zmq.PUSH)
osock.connect("tcp://127.0.0.1:6706")
osock.send_json(doc)
def accept_connection(doc):
"""the queue responsible accepting connections"""
zcontext = zmq.Context()
osock = zcontext.socket(zmq.PUSH)
osock.connect("tcp://127.0.0.1:6707")
osock.send_json(doc)
def add_profile_pic(doc):
"""Add the profile pic of the user"""
proxy = Server('http://localhost:7006')
answer = proxy.profile_pic(doc)
return answer
def search(doc):
"""search the user in the database"""
proxy = Server('http://localhost:7006')
answer = proxy.search(doc)
return answer
def profile(doc):
"""search the user in the database"""
proxy = Server('http://localhost:7006')
answer = proxy.profile(doc)
return answer
async def get_answer(aphorism, content):
"""Return the response to particular question"""
# time.sleep(0.0)
# fetch responsible function
# function = aphorisms.get(aphorism, "Error:Unknown aphorism.")
function = eval(aphorism)
answer = function(content)
return send(answer)
def send(data):
"""Prepare the data to be sent via socket"""
json_data = json.dumps(data)
data_bytes = json_data.encode()
return data_bytes
def parse_command_line(description):
"""arse command line and return a socket address."""
parser = argparse.ArgumentParser(description=description)
parser.add_argument('host', help="IP or hostname")
parser.add_argument("-p", metavar='port', type=int, default=1060, help="TCP port (default 1060)")
args = parser.parse_args()
address = (args.host, args.p)
return address
def recv_untill(sock, suffix):
"""Receive bytes over socket `sock` until we receive the `suffix`."""
message = sock.recv(4096)
if not message:
raise EOFError("Socket closed")
while not message.endswith(suffix):
data = sock.recv(4096)
if not data:
raise IOError('received {!r} then socket closed'.format(message))
message += data
return message

Related

python coroutine asyncio/ await / aiohttp

new in asyncio world.
going straight to the point...
I want to do/make a request(aiohttp) to a site.
if the wait for an answer pass than N seconds I want to stop the process of waiting.
Do the process again by setting a limit of attempts if needed.
async def search_skiping_add(name_search):
start_time = time()
async with aiohttp.ClientSession() as session:
url = f'https://somesitehere.com'
r = await session.get(url)
final_time = time()
result_time =round(final_time-start_time)
print(result_time)
Maybe, I know, have some way to do it synchronously, but it's an excuse to start using asyncio somehow too.
This should give you an idea of how to use async with aiohttp:
from aiohttp import ClientSession
from asyncio import gather, create_task, sleep, run
from traceback import format_exc
def returnPartionedList(inputlist: list, x: int = 100) -> list: # returns inputlist split into x parts, default is 100
return([inputlist[i:i + x] for i in range(0, len(inputlist), x)])
# You could change validate to an integer and thereby increasing the retry count as needed.
async def GetRessource(url: str, session: ClientSession, validate: bool = False) -> dict:
try:
async with session.get(url) as response:
if response.status == 200:
r: dict = await response.json() # Set equal to .text() to get results as a string
return(r)
else:
r: str = await response.text()
if not validate:
await sleep(3) # Sleep for x amount of seconds before retry
return(await GetRessource(url, session, True))
print(f"Error, got response code: {response.status} message: {r}")
except Exception:
print(f"General Exception:\n{format_exc()}")
return({})
async def GetUrls(urls: list) -> list:
resultsList: list = []
UrlPartitions: list = returnPartionedList(urls, 20) # Rate limit to 20 requests per loop
async with ClientSession(timeout=15) as session: # Timeout is equal to the time to wait in seconds before terminating, default is 300 seconds or 5 minutes.
for partition in UrlPartitions:
partitionTasks: list = [create_task(GetRessource(url, session)) for url in partition]
resultsList.append(await gather(*partitionTasks, return_exceptions=False))
return(resultsList) # Or you can do more processing here before returning
async def main():
urls: list = ["...", "...", "..."] # list of urls to get from
results: list = await GetUrls(urls)
print(results)
if __name__ == "__main__":
run(main())

discord.py not finding commands within a class

If I use .join in discord with the code below, I get the following error:
Ignoring exception in command None:
discord.ext.commands.errors.CommandNotFound: Command "join" is not found
I'm not sure how to fix this (this is turning into being part of a music bot). Here is the relevant code:
import discord
import discord.voice_client
from discord import utils
from discord.ext import commands
from discord.utils import *
import discord.utils
import lavalink
bot = commands.Bot(command_prefix= ".", intents=intents)
class MusicCog(commands.Cog):
def __init__(self, bot):
self.bot = bot
self.bot.music = lavalink.Client(self.bot.user.id)
self.bot.music.add_node('localhost', 7000, 'testing', 'na' , 'music-node')
self.bot.add_listener(self.bot.music.voice_update_handler, 'on-socket-response')
self.bot.music.add_event_hook(self.track_hook)
#commands.command(name= 'Join')
async def join(self, ctx):
print("join command worked")
member = utils.find(lambda m: m.id == ctx.author.id, ctx.guild.members)
if member is not None and member.voice is not None:
vc = member.voice.channel
player = self.bot.music.player_manager.create(ctx.guild.id, endpoint= str(ctx.guild.region))
if not player.is_connected:
player.store('channel', ctx.channel.id)
await self.connect_to(ctx.guild.id, str(vc.id))
#commands.command(name= "Play")
async def play(self, ctx, *, query):
try:
player = self.bot.music.player_manager.get(ctx.guild.id)
query = f'ytsearch: {query}'
results = await player.node.get_tracks(query)
tracks = results['tracks'][0:10]
i = 0
query_result = ''
for track in tracks:
i = i + 1
query_result = query_result + f'{i}) {track["info"]["title"]} - {track["info"]["url"]}\n'
show_songs = discord.Embed(
title= None,
description= None,
colour= discord.Colour.blue()
)
show_songs.description = query_result
await ctx.channel.send(embed= show_songs)
def check(m):
return m.author.id == ctx.author.id
response = await self.bot.wait_for('message', check=check)
track = tracks[int(response.conetent)-1]
player.add(requester = ctx.author.id, track = track)
if not player.is_playing:
await player.play()
except Exception as error:
print(error)
async def track_hook(self, event):
if isinstance(event, lavalink.events.QueueEndEvent):
guild_id = int(event.player.guild.id)
await self.connect_to(guild_id, None)
async def connect_to(self, guild_id: int, channel_id: str):
ws = self.bot._connection._get_websocket(guild_id)
await ws.voice_state(str(guild_id), channel_id)
def setup(bot):
bot.add_cog(MusicCog(bot))
I've tried changing #commands.command to #bot.command, which didn't end up working, and I'm almost certain the indentation is correct throughout the code, so at this point, I'm not sure how to fix it. Any help would be appreciated!
By default, commands are case sensitive so you'd have to type .Join or .Play in order to invoke your them.
If you want your commands to be case insensitive, simply type :
bot = commands.Bot(command_prefix='.', intents=intents, case_insensitive=True)
PS : the name argument in the commands.command() decorator refers to what needs to be typed after your prefix to invoke the function.

Change log level dynamically from asyncio outside module

I need to dynamically change log level without reloading any modules. For this, I am using a combination of asyncio and multiprocessing (both for the bigger program that I am writing).
log_config.py
def setupLogging(log_level=None): # load custom logger
with open(os.path.join(LOG_PATH,'log_config.yaml'), 'rt') as file_:
config = yaml.safe_load(file_.read())
logging.config.dictConfig(config)
logging.Formatter.converter = time.gmtime
if not log_level:
log_level = 'preview'
return logging.getLogger(log_level)
logger=setupLogging() # global var that needs dynamic updation
async def logger_changes(): # socket listener that changes the logger object
global logger
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('localhost', 9999))
sock.listen()
sock.setblocking(0)
while True:
try:
conn, addr = sock.accept()
data = conn.recv(1024)
conn.send(data)
logger = setupLogging(data.decode('utf8'))
except Exception as e:
pass
await asyncio.sleep(5)
async def logger_handler():
t1 = asyncio.create_task(logger_changes())
await t1
def start_logger():
start_func = asyncio.run(logger_handler())
pLOGGER = Process(name="__startLOGGER__", target=start_logger, daemon=False)
pLOGGER.start()
Main Functionality
import log_config
logger=log_config.logger
async def core_func():
for i in range(10):
logger.debug("sample debug log")
logger.info("sample info log")
logger.warning("Watch out!")
logger.error("Heading for trouble!")
logger.critical("Seriously, do something!")
print("#"*80)
await asyncio.sleep(5)
async def core_func_parent():
t1 = asyncio.create_task(core_func())
await t1
def core_func_handler():
start_func = asyncio.run(core_func_parent())
if __name__=='__main__':
pMODULE = Process(name="__startMODULE__", target=core_func_handler, daemon=False)
pMODULE.start()
Trigger Log Level Change
import socket
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.connect(('localhost', 9999))
request = "test"
try:
server.send(request.encode('utf8'))
response = server.recv(255).decode('utf8')
print(response)
except Exception as e:
print(e)
server.close()
Steps
Run main functionality python script. Automatically invoked log_config from inside, starts the socket listener and instantiates the logger object
Run trigger log level change that sends a message to the web socket with the updated logger setting
Challenge
Once I run step 1, the logging level is INFO (which is exactly what I want)
But once I run step 2, I am expecting the logging level to change to DEBUG which is not happening.
Any help is appreciated.

How do I represent a two-dimensional array in Protocol Buffers?

for example:
[[1,2],[3,4]...]
I just wanted to test if RPC supported a two-dimensional array, But there is something wrong, I am following the official docs.
The server is as follows:
data = [[i, 9] for i in range(128)]
class Greeter(hello_pb2_grpc.GreeterServicer):
def SayHello(self, request, context):
return hello_pb2.HelloReply(results=data)
def serve():
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10), options=[
(cygrpc.ChannelArgKey.max_send_message_length, -1),
(cygrpc.ChannelArgKey.max_receive_message_length, -1)
])
hello_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server)
server.add_insecure_port('[::]:50052')
server.start()
try:
while True:
time.sleep(_ONE_DAY_IN_SECONDS)
except KeyboardInterrupt:
server.stop(0)
if __name__ == '__main__':
serve()
The client is as follows:
def insecure_channel(host, port):
channel = grpc.insecure_channel(
target=host if port is None else '%s:%d' % (host, port),
options=[(cygrpc.ChannelArgKey.max_send_message_length, -1),
(cygrpc.ChannelArgKey.max_receive_message_length, -1)])
return grpc.beta.implementations.Channel(channel)
def run():
channel = grpc.insecure_channel('localhost:50052')
stub = hello_pb2_grpc.GreeterStub(channel)
st = time.time()
response = stub.SayHello(hello_pb2.HelloRequest(name='test'))
et = time.time() - st
print("Greeter client received: {}, {}".format(type(response.results), et))
if __name__ == '__main__':
run()
The protobuf definition is as follows:
syntax = "proto3";
package hello;
service Greeter {
rpc SayHello (HelloRequest) returns (HelloReply) {}
}
message HelloRequest {
string name = 1;
}
message HelloReply {
repeated Result results = 1;
}
message Result {
repeated int32 index = 1;
repeated int32 count = 2;
}
But I get the error like this:
File "greeter_server.py", line 19, in SayHello
return hello_pb2.HelloReply(results=data)
TypeError: Parameter to MergeFrom() must be instance of same class: expected hello.Result got list.
I am using python3.
Thanks in advance.
As per your question, [[1,2],[3,4]...] is an array of arrays.
Something like this should solve your problem:
message InternalArray {
repeated int internal_array = 1;
}
repeated InternalArray array = 1;
Problem is a list you are passing where hello.Result is expected.
Use:
import hello_pb2
to_return = hello_pb2.Result()
to_return.index = 111
to_return.count = 222
now return `to_return` from rpc call.

autobahn mosquitto server chat

I´m trying to connect to a mosquitto broker using autobahn python.
If I use sub.py that has this code inside:
import mosquitto
def on_connect(mosq, obj, rc):
print("rc: "+str(rc))
def on_message(mosq, obj, msg):
print(msg.topic+" "+str(msg.qos)+" "+str(msg.payload))
def on_publish(mosq, obj, mid):
print("mid: "+str(mid))
def on_subscribe(mosq, obj, mid, granted_qos):
print("Subscribed: "+str(mid)+" "+str(granted_qos))
def on_log(mosq, obj, level, string):
print(string)
mqttc = mosquitto.Mosquitto()
mqttc.on_message = on_message
mqttc.on_connect = on_connect
mqttc.on_publish = on_publish
mqttc.on_subscribe = on_subscribe
# Uncomment to enable debug messages
mqttc.on_log = on_log
mqttc.connect("localhost", 1883, 60)
mqttc.subscribe("control", 0)
rc = 0
while rc == 0:
rc = mqttc.loop()
print("rc: "+str(rc))
it is connecting to the broker and retrieving all messages that a client publish to control channel.
I´d like to push somehow those messages using websockets to a webpage for that I am trying to use websocket autobahn py and modify the example from here http://autobahn.ws/python/getstarted#yourfirstserver
My code is like this
import sys
import mosquitto
from twisted.internet import reactor
from twisted.python import log
from autobahn.websocket import WebSocketServerFactory, \
WebSocketServerProtocol, \
listenWS
class EchoServerProtocol(WebSocketServerProtocol):
def on_connect(mosq, obj, rc):
print("rc: "+str(rc))
def on_message(mosq, obj, msg):
print(msg.topic+" "+str(msg.qos)+" "+str(msg.payload))
def onMessage(self, msg, binary):
print "sending echo:", msg
self.sendMessage(msg, binary)
def on_publish(mosq, obj, mid):
print("mid: "+str(mid))
def on_subscribe(mosq, obj, mid, granted_qos):
print("Subscribed: "+str(mid)+" "+str(granted_qos))
mqttc = mosquitto.Mosquitto()
mqttc.on_message = on_message
mqttc.on_connect = on_connect
mqttc.on_publish = on_publish
mqttc.on_subscribe = on_subscribe
# Uncomment to enable debug messages
mqttc.on_log = on_log
mqttc.connect("192.168.2.109", 1883, 60)
mqttc.subscribe("control", 0)
rc = 0
while rc == 0:
rc = mqttc.loop()
print("rc: "+str(rc))
if __name__ == '__main__':
log.startLogging(sys.stdout)
factory = WebSocketServerFactory("ws://192.168.2.109:8899", debug = TRUE)
factory.protocol = EchoServerProtocol
listenWS(factory)
reactor.run()
but I receive this error when I try to run it:
root#Ubuntu:~/authobahn# python myserver.py Traceback (most recent
call last): File "myserver.py", line 30, in
mqttc.on_message = on_message NameError: name 'on_message' is not defined
The problem is that you have defined your on_message() function inside the EchoServerProtocol class. This means it is not visible to the global mqttc variable. You probably want to put all of the mqttc code inside your class as well, although it depends on what you actually want to achieve.
You could do something like the code below:
import mosquitto
class MyMQTTClass:
def __init__(self, clientid=None):
self._mqttc = mosquitto.Mosquitto()
self._mqttc.on_message = self.mqtt_on_message
self._mqttc.on_connect = self.mqtt_on_connect
self._mqttc.on_publish = self.mqtt_on_publish
self._mqttc.on_subscribe = self.mqtt_on_subscribe
def mqtt_on_connect(self, mosq, obj, rc):
print("rc: "+str(rc))
def mqtt_on_message(self, mosq, obj, msg):
print(msg.topic+" "+str(msg.qos)+" "+str(msg.payload))
def mqtt_on_publish(self, mosq, obj, mid):
print("mid: "+str(mid))
def mqtt_on_subscribe(self, mosq, obj, mid, granted_qos):
print("Subscribed: "+str(mid)+" "+str(granted_qos))
def mqtt_on_log(self, mosq, obj, level, string):
print(string)
def run(self):
self._mqttc.connect("test.mosquitto.org", 1883, 60)
self._mqttc.subscribe("$SYS/#", 0)
rc = 0
while rc == 0:
rc = self._mqttc.loop()
return rc
# If you want to use a specific client id, use
# mqttc = MyMQTTClass("client-id")
# but note that the client id must be unique on the broker. Leaving the client
# id parameter empty will generate a random id for you.
mqttc = MyMQTTClass()
rc = mqttc.run()
print("rc: "+str(rc))

Resources