why is asynchronous slower than synchronous? - python-asyncio

Why is it that when I start testing the load on the server, the synchronous writing style works hundreds of times faster than the asynchronous one. I'm just trying my hand at asynchrony, what am I doing wrong?
ab -t 10 -n 10000 -c 100 http://127.0.0.1:8888/api/db/sync/
Number of requests per second: 35
ab -t 10 -n 10000 -c 100 http://127.0.0.1:8888/api/db/
Number of requests per second: 0.37
import asyncio
import json
import time
from asyncio import sleep
from typing import Optional, Union
import tornado.ioloop
import tornado.web
from jinja2 import Environment, select_autoescape, FileSystemLoader
from sqlalchemy import text, create_engine
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
from sqlalchemy.orm import sessionmaker, Session
from tornado import gen
from tornado.httpserver import HTTPServer
env = Environment(
loader=FileSystemLoader('templates'),
autoescape=select_autoescape()
)
async_engine = create_async_engine(
"postgresql+asyncpg://***:***#localhost:5431/***",
echo=True,
)
async_session = sessionmaker(
async_engine, expire_on_commit=False, class_=AsyncSession
)
engine = create_engine(
"postgresql://***:***#localhost:5431/***",
echo=True,
)
sync_session = sessionmaker(
engine, expire_on_commit=False
)
class ApiDbHandler(tornado.web.RequestHandler):
def prepare(self):
header = "Content-Type"
body = "application/json"
self.set_header(header, body)
async def get(self):
async with async_session() as session:
session: AsyncSession
for result in await asyncio.gather(
session.execute(text('SELECT * from *** where client_id = :bis_id').bindparams(bis_id='bis1')),
session.execute(text('SELECT * from *** where bis_id = :bis_id').bindparams(bis_id='bis1')),
session.execute(text('SELECT * from *** where abs_id = :abs_id').bindparams(abs_id='abs1')),
session.execute(
text('SELECT * from *** where abs_id = :abs_id').bindparams(abs_id='abs1')),
):
self.write(
json.dumps(result.fetchall(), default=str)
)
class ApiDbSyncHandler(tornado.web.RequestHandler):
def prepare(self):
header = "Content-Type"
body = "application/json"
self.set_header(header, body)
def get(self):
with sync_session() as session:
session: Session
assets = session.execute(text('SELECT * from *** where client_id = :bis_id').bindparams(bis_id='bis1'))
self.write(
json.dumps(assets.fetchall(), default=str)
)
ili = session.execute(text('SELECT * from *** where bis_id = :bis_id').bindparams(bis_id='bis1'))
self.write(
json.dumps(ili.fetchall(), default=str)
)
securities = session.execute(text('SELECT * from *** where abs_id = :abs_id').bindparams(abs_id='abs1'))
self.write(
json.dumps(securities.fetchall(), default=str)
)
abs_accounts = session.execute(text('SELECT * from *** where abs_id = :abs_id').bindparams(abs_id='abs1'))
self.write(
json.dumps(abs_accounts.fetchall(), default=str)
)
def make_app():
return tornado.web.Application([
(r"/api/db/", ApiDbHandler),
(r"/api/db/sync/", ApiDbSyncHandler),
])
if __name__ == "__main__":
app = make_app()
server = HTTPServer(app)
server.listen(8888)
tornado.ioloop.IOLoop.current().start()

Related

Websocket with Locust, no error but no data

The fixed code (from https://medium.com/#rajatsaxena120/websockets-in-python-d91c7bc2fd22, https://stackoverflow.com/questions/ask?newreg=520c9b343e534667aa88f67ea3f79cb4) seems to work:
import time, websocket
from locust import HttpUser, task, between, events
from websocket import create_connection
import gevent
class QuickstartUser(HttpUser):
wait_time = between(1, 5)
#task
def on_start(self):
ws = create_connection('wss://REDACTED.com')
g = gevent.spawn(self.connect)
g.get(block=True, timeout=10)
g = gevent.spawn(self.subscribe)
g.get(block=True, timeout=10)
g = gevent.spawn(self.send)
g.get(block=True, timeout=10)
def _receive():
ws = create_connection('wss://REDACTED.com')
while True:
res = ws.recv()
events.request_success.fire(
request_type='Websocket Receive Message',
name='test websocket message receive',
response_time=0,
response_length=len(res)
)
gevent.spawn(_receive)
But when running tests, nothing happen, no connection to websocket, no load on the server.
What am I missing?

Python Watchdog Custom Event Handler Not Firing Correctly

I'm using Python's watchdog module to listen for created events on a certain directory. Where I do some processing on newly created .csv files. When I test my code with nothing in the handler the watchdog fires correctly for all pasted/created files, but when I add my code/logic inside the handler it fires less than expected (52/60 files for example).
OS used: Windows 10, code is expected to work on a Windows server.
Python version: 3.7.3
Code:
import time
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
import pandas as pd
import numpy as np
from error_handling import ErrorHandler
import os
from timeloop import Timeloop
from datetime import timedelta
import json
from fill_data import OnMyWatch2
class OnMyWatch:
# Set the directory on watch
watchDirectory = "."
def __init__(self):
self.observer1 = Observer()
def run(self):
self.observer1.schedule(Handler() , self.watchDirectory, recursive = True)
self.observer1.start()
try:
while True:
time.sleep(1)
except:
self.observer1.stop()
print("Observer Stopped")
self.observer1.join()
class Handler(FileSystemEventHandler):
#staticmethod
def on_any_event(event):
if event.is_directory:
return None
elif event.event_type == 'created':
try:
# Event is created, you can process it now
print("Watchdog received created event - % s." % event.src_path)
pathlower = str(event.src_path).lower()
if ".csv" in pathlower:
print("File is csv file")
# Once any code is added here the problem happens
# Example:
# df = pd.read_csv(event.src_path, names= ALL_INI.columnlistbundle.split("|"), sep="|", encoding='latin-1', engine='python')
# arraySelectedColumnsBundle = ALL_INI.selectedcolumnsbundle.split(",")
# bundle_df = df[np.array(arraySelectedColumnsBundle)]
else:
print("File is not csv file")
except Exception as e:
ErrorHandler(0, 'In Observer ', '-7', 'Exception ' + str(e), '', 1, '')
if __name__ == '__main__':
if os.path.isdir("."):
watch = OnMyWatch()
watch.run()

How to use python-telegram-bot repository to send message to telegram user request?

I programmed a telegram bot for downloading torrents with python-telegram-bot repository; however, my functions seem to work on the server side and the responses appear on my terminal/console where im running the py-script and transmissionBT server, but it does not send the response back to the user. below is my code:
def get_torrent(MAGNET):
import subprocess
get_it = subprocess.run(["transmission-remote", "-a", "-D", "-o", "--utp", MAGNET])
return "*success*"
def list_torrent(LISTED):
import subprocess
get_list = subprocess.run(["transmission-remote", LISTED])
return "*listed*"
def del_torrent(TORRENT_NO):
import subprocess
del_torrent = subprocess.run(["transmission-remote", "-t", TORRENT_NO, "-r"])
return "*deleted*"
def crepttit(bot, update, args):
import time
MAGNET = " ".join(args)
media_content = get_torrent(MAGNET)
time.sleep(1)
bot.send_meessage(chat_id=update.message.chat_id, text=media_content)
def crepttl(bot, update, args):
import time
LISTED = " ".join(args)
listed_torrent = list_torrent(LISTED)
time.sleep(1)
chat_id = update.message.chat_id
bot.send_message(chat_id=chat_id, text=torrent_list)
def crepttd(bot, update, args):
import time
TORRENT_NO = " ".join(args)
deleted_torrent = del_torrent(TORRENT_NO)
time.sleep(1)
bot.send_meessage(chat_id=update.message.chat_id, text=deleted_torrent)
from telegram.ext import Updater, CommandHandler
import subprocess
import time
import os
TOKEN = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
updater = Updater(TOKEN)
dispatcher = updater.dispatcher
start_handler = CommandHandler("creptt", crepttit, pass_args=True)
dispatcher.add_handler(start_handler)
start_handler2 = CommandHandler("crepttl", crepttl, pass_args=True)
dispatcher.add_handler(start_handler2)
start_handler3 = CommandHandler("crepttd", crepttd, pass_args=True)
dispatcher.add_handler(start_handler3)
updater.start_polling()
updater.idle()

http command : SyntaxError: invalid syntax on file core.py

I am trying to install and run http command in my Ubuntu 18.04.4 LTS.
I think I have successfully installed the http. But when I am going to check the version of http by http --version or trying to connect to a website, say http github.org I am getting the following:
Traceback (most recent call last):
File "/usr/local/bin/http", line 8, in <module>
sys.exit(main())
File "/usr/local/lib/python2.7/dist-packages/httpie/__main__.py", line 10, in main
from .core import main
File "/usr/local/lib/python2.7/dist-packages/httpie/core.py", line 21
args: List[Union[str, bytes]] = sys.argv,
^
SyntaxError: invalid syntax
Here is how my main.py looks like:
#!/usr/bin/env python
"""The main entry point. Invoke as `http' or `python -m httpie'.
"""
import sys
def main():
try:
from .core import main
exit_status = main()
except KeyboardInterrupt:
from httpie.status import ExitStatus
exit_status = ExitStatus.ERROR_CTRL_C
sys.exit(exit_status.value)
if __name__ == '__main__':
main()
And core.py in python 2.7
import argparse
import os
import platform
import sys
from typing import List, Union
import requests
from pygments import __version__ as pygments_version
from requests import __version__ as requests_version
from httpie import __version__ as httpie_version
from httpie.client import collect_messages
from httpie.context import Environment
from httpie.downloads import Downloader
from httpie.output.writer import write_message, write_stream
from httpie.plugins import plugin_manager
from httpie.status import ExitStatus, http_status_to_exit_status
def main(
args: List[Union[str, bytes]] = sys.argv,
env=Environment(),
) -> ExitStatus:
"""
The main function.
Pre-process args, handle some special types of invocations,
and run the main program with error handling.
Return exit status code.
"""
program_name, *args = args
env.program_name = os.path.basename(program_name)
args = decode_raw_args(args, env.stdin_encoding)
plugin_manager.load_installed_plugins()
from httpie.cli.definition import parser
if env.config.default_options:
args = env.config.default_options + args
include_debug_info = '--debug' in args
include_traceback = include_debug_info or '--traceback' in args
if include_debug_info:
print_debug_info(env)
if args == ['--debug']:
return ExitStatus.SUCCESS
exit_status = ExitStatus.SUCCESS
try:
parsed_args = parser.parse_args(
args=args,
env=env,
)
except KeyboardInterrupt:
env.stderr.write('\n')
if include_traceback:
raise
exit_status = ExitStatus.ERROR_CTRL_C
except SystemExit as e:
if e.code != ExitStatus.SUCCESS:
env.stderr.write('\n')
if include_traceback:
raise
exit_status = ExitStatus.ERROR
else:
try:
exit_status = program(
args=parsed_args,
env=env,
)
except KeyboardInterrupt:
env.stderr.write('\n')
if include_traceback:
raise
exit_status = ExitStatus.ERROR_CTRL_C
except SystemExit as e:
if e.code != ExitStatus.SUCCESS:
env.stderr.write('\n')
if include_traceback:
raise
exit_status = ExitStatus.ERROR
except requests.Timeout:
exit_status = ExitStatus.ERROR_TIMEOUT
env.log_error(f'Request timed out ({parsed_args.timeout}s).')
except requests.TooManyRedirects:
exit_status = ExitStatus.ERROR_TOO_MANY_REDIRECTS
env.log_error(
f'Too many redirects'
f' (--max-redirects=parsed_args.max_redirects).'
)
except Exception as e:
# TODO: Further distinction between expected and unexpected errors.
msg = str(e)
if hasattr(e, 'request'):
request = e.request
if hasattr(request, 'url'):
msg = (
f'{msg} while doing a {request.method}'
f' request to URL: {request.url}'
)
env.log_error(f'{type(e).__name__}: {msg}')
if include_traceback:
raise
exit_status = ExitStatus.ERROR
return exit_status
def program(
args: argparse.Namespace,
env: Environment,
) -> ExitStatus:
"""
The main program without error handling.
"""
exit_status = ExitStatus.SUCCESS
downloader = None
try:
if args.download:
args.follow = True # --download implies --follow.
downloader = Downloader(
output_file=args.output_file,
progress_file=env.stderr,
resume=args.download_resume
)
downloader.pre_request(args.headers)
initial_request = None
final_response = None
for message in collect_messages(args, env.config.directory):
write_message(
requests_message=message,
env=env,
args=args,
)
if isinstance(message, requests.PreparedRequest):
if not initial_request:
initial_request = message
else:
final_response = message
if args.check_status or downloader:
exit_status = http_status_to_exit_status(
http_status=message.status_code,
follow=args.follow
)
if (not env.stdout_isatty
and exit_status != ExitStatus.SUCCESS):
env.log_error(
f'HTTP {message.raw.status} {message.raw.reason}',
level='warning'
)
if downloader and exit_status == ExitStatus.SUCCESS:
# Last response body download.
download_stream, download_to = downloader.start(
initial_url=initial_request.url,
final_response=final_response,
)
write_stream(
stream=download_stream,
outfile=download_to,
flush=False,
)
downloader.finish()
if downloader.interrupted:
exit_status = ExitStatus.ERROR
env.log_error(
'Incomplete download: size=%d; downloaded=%d' % (
downloader.status.total_size,
downloader.status.downloaded
))
return exit_status
finally:
if downloader and not downloader.finished:
downloader.failed()
if (not isinstance(args, list) and args.output_file
and args.output_file_specified):
args.output_file.close()
def print_debug_info(env: Environment):
env.stderr.writelines([
f'HTTPie {httpie_version}\n',
f'Requests {requests_version}\n',
f'Pygments {pygments_version}\n',
f'Python {sys.version}\n{sys.executable}\n',
f'{platform.system()} {platform.release()}',
])
env.stderr.write('\n\n')
env.stderr.write(repr(env))
env.stderr.write('\n')
def decode_raw_args(
args: List[Union[str, bytes]],
stdin_encoding: str
) -> List[str]:
"""
Convert all bytes args to str
by decoding them using stdin encoding.
"""
return [
arg.decode(stdin_encoding)
if type(arg) == bytes else arg
for arg in args
]
I have two versions of python are installed in my system. As I understood python2.7 came as pre-installed and I have installed python 3.
Currently the default python version is python 3.
What could cause the error? Is it with my different python version or syntax error?

Error in Post Requst (Kusto & Databricks)

I'm trying to follow the example code here:
Here is my code:
import com.microsoft.kusto.spark.datasource.KustoOptions
import com.microsoft.kusto.spark.sql.extension.SparkExtension._
import org.apache.spark.SparkConf
import org.apache.spark.sql._
val cluster = dbutils.secrets.get(scope = "key-vault-secrets", key = "ClusterName")
val client_id = dbutils.secrets.get(scope = "key-vault-secrets", key = "ClientId")
val client_secret = dbutils.secrets.get(scope = "key-vault-secrets", key = "ClientSecret")
val authority_id = dbutils.secrets.get(scope = "key-vault-secrets", key = "TenantId")
val database = "db"
val table = "tablename"
val conf: Map[String, String] = Map(
KustoOptions.KUSTO_AAD_CLIENT_ID -> client_id,
KustoOptions.KUSTO_AAD_CLIENT_PASSWORD -> client_secret,
KustoOptions.KUSTO_QUERY -> s"$table | top 100"
)
// Simplified syntax flavor
import org.apache.spark.sql._
import com.microsoft.kusto.spark.sql.extension.SparkExtension._
import org.apache.spark.SparkConf
val df = spark.read.kusto(cluster, database, "", conf)
display(df)
However this gives me this error:
com.microsoft.azure.kusto.data.exceptions.DataServiceException: Error in post request
at com.microsoft.azure.kusto.data.Utils.post(Utils.java:106)
at com.microsoft.azure.kusto.data.ClientImpl.execute(ClientImpl.java:89)
at com.microsoft.azure.kusto.data.ClientImpl.execute(ClientImpl.java:45)
at com.microsoft.kusto.spark.utils.KustoDataSourceUtils$.getSchema(KustoDataSourceUtils.scala:103)
at com.microsoft.kusto.spark.datasource.KustoRelation.getSchema(KustoRelation.scala:102)
at com.microsoft.kusto.spark.datasource.KustoRelation.schema(KustoRelation.scala:36)
at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:450)
at org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:297)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:283)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:201)
at com.microsoft.kusto.spark.sql.extension.SparkExtension$DataFrameReaderExtension.kusto(SparkExtension.scala:19)
at linef172a4a7eaa6435fa4ff9fec071cf03535.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(command-1810687702746193:25)
at linef172a4a7eaa6435fa4ff9fec071cf03535.$read$$iw$$iw$$iw$$iw$$iw.<init>(command-1810687702746193:86)
at linef172a4a7eaa6435fa4ff9fec071cf03535.$read$$iw$$iw$$iw$$iw.<init>(command-1810687702746193:88)
at linef172a4a7eaa6435fa4ff9fec071cf03535.$read$$iw$$iw$$iw.<init>(command-1810687702746193:90)
at linef172a4a7eaa6435fa4ff9fec071cf03535.$read$$iw$$iw.<init>(command-1810687702746193:92)
at linef172a4a7eaa6435fa4ff9fec071cf03535.$read$$iw.<init>(command-1810687702746193:94)
at linef172a4a7eaa6435fa4ff9fec071cf03535.$read.<init>(command-1810687702746193:96)
at linef172a4a7eaa6435fa4ff9fec071cf03535.$read$.<init>(command-1810687702746193:100)
at linef172a4a7eaa6435fa4ff9fec071cf03535.$read$.<clinit>(command-1810687702746193)
at linef172a4a7eaa6435fa4ff9fec071cf03535.$eval$.$print$lzycompute(<notebook>:7)
Any ideas?
Make sure that the format of your cluster name matches the expected format.
The expected is clustername.region
In the file you are reading your query from, make sure you append a line break ( \n ) at the end of each line.

Resources