import asyncio
import ib_insync as ibi
import symbol_list
import time
start = time.perf_counter()
stocklist = symbol_list.test
endDateTime = '20190328 09:30:00'
durationStr='1 D'
dataDirectory = './data/tmp'
class App:
async def run(self):
self.ib = ibi.IB()
with await self.ib.connectAsync():
contracts = [
ibi.Stock(symbol, 'SMART', 'USD')
for symbol in ['AAPL', 'TSLA', 'AMD', 'INTC']]
for contract in contracts:
# self.ib.reqMktData(contract)
bars = await self.ib.reqHistoricalDataAsync(contract,
endDateTime=endDateTime, durationStr=durationStr,
barSizeSetting='5 mins', whatToShow='MIDPOINT', useRTH=True)
df = ibi.util.df(bars)
df.to_csv(f"{dataDirectory}/{contract.symbol}.csv")
async for tickers in self.ib.pendingTickersEvent:
for ticker in tickers:
print(ticker)
def stop(self):
self.ib.disconnect()
app = App()
try:
asyncio.run(app.run())
except (KeyboardInterrupt, SystemExit):
app.stop()
endtime = (time.perf_counter() - start)/60
print(f"Process time: {endtime:,.2f} minutes")
Please help. I modified the example code from async-streaming-example. I didn't get any error message, but it just runs without giving me the shell prompt. And this code should only take less than a minute, if it runs properly. Essentially, instead of reqMktData, I want to use reqHistoricalDataAsync to get historical data, asynchronously. I've also looked at async execution with ib_insync, but I wasn't able to get that technique to work, either. Could you show me what I'm doing wrong? I welcome any async solutions. Thank you.
Related
My framework (Locust, https://github.com/locustio/locust) is based on gevent and greenlets. But I would like to leverage Playwright (https://playwright.dev/python/), which is built on asyncio.
Naively using Playwrights sync api doesnt work and gives an exception:
playwright._impl._api_types.Error: It looks like you are using Playwright Sync API inside the asyncio loop.
Please use the Async API instead.
I'm looking for some kind of best practice on how to use async in combination with gevent.
I've tried a couple different approaches but I dont know if I'm close or if what I'm trying to do is even possible (I have some experience with gevent, but havent really used asyncio before)
Edit: I kind of have something working now (I've removed Locust and just directly spawned some greenlets to make it easier to understan). Is this as good as it gets, or is there a better solution?
import asyncio
import threading
from playwright.async_api import async_playwright
import gevent
def thr(i):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(do_stuff(i))
loop.close()
async def do_stuff(i):
playwright = await async_playwright().start()
browser = await playwright.chromium.launch(headless=False)
page = await browser.new_page()
await page.wait_for_timeout(5000)
await page.goto(f"https://google.com")
await page.close()
print(i)
def green(i):
t = threading.Thread(target=thr, args=(i,))
t.start()
# t.join() # joining doesnt work, but I couldnt be bothered right now :)
g1 = gevent.spawn(green, 1)
g2 = gevent.spawn(green, 2)
g1.join()
g2.join()
Insipred by #user4815162342 's comment, I went with something like this:
from playwright.async_api import async_playwright # need to import this first
from gevent import monkey, spawn
import asyncio
import gevent
monkey.patch_all()
loop = asyncio.new_event_loop()
async def f():
print("start")
playwright = await async_playwright().start()
browser = await playwright.chromium.launch(headless=True)
context = await browser.new_context()
page = await context.new_page()
await page.goto(f"https://www.google.com")
print("done")
def greeny():
while True: # and not other_exit_condition
future = asyncio.run_coroutine_threadsafe(f(), loop)
while not future.done():
gevent.sleep(1)
greenlet1 = spawn(greeny)
greenlet2 = spawn(greeny)
loop.run_forever()
The actual implementation will end up in Locust some day, probably after some optimization (reusing browser instance etc)
Here's a simple way to integrate asyncio and gevent:
Run an asyncio loop in a dedicated thread
Use asyncio.run_coroutine_threadsafe() to run a coroutine
Use gevent.event.Event to wait until the coroutine resolves
import asyncio
import threading
import gevent
loop = asyncio.new_event_loop()
loop_thread = threading.Thread(target=loop.run_forever, daemon=True)
loop_thread.start()
async def your_coro():
# ...
def wait_until_complete(coro):
future = asyncio.run_coroutine_threadsafe(coro, loop)
event = gevent.event.Event()
future.add_dome_callback(lambda _: event.set())
event.wait()
return future.result()
result = wait_until_complete(your_coro())
I made a discord bot that shows you gifs when you type a certain command but the problem is that it works fine in the first half but takes a long time to show the gifs when not used.
Basically it doesn't show the gifs instantly when not used.
Here's the code that I've written:
#client.command()
async def gif(ctx, *, q = 'dance'):
api_key = 'Some Key here'
api_instanc = giphy_client.DefaultApi()
try:
api_responce = api_instanc.gifs_search_get(api_key, q, limit = 7,rating = 'r')
lst = list(api_responce.data)
giff = random.choice(lst)
emb = discord.Embed(title = f"Requested by {ctx.author} " + q )
emb.set_image(url= f'https://media.giphy.com/media/{giff.id}/giphy.gif')
await ctx.channel.send(embed = emb)
except ApiException as e:
await ctx.channel.send("API EXCEPTION")
It doesn't show any errors but doesn't work after the long time.
Any re-write of the code with aiohttp will be appreciated because I am learning that.
I think the module you are using is not asynchronous which leads to blocking read more.
Default in the command is search = None you can use that with an if statement to check.
After that is the request for the api to get the image.
Here is the code edited to use aiohttp
# import aiohttp
# import random
#bot.command()
async def giphy(ctx, search: str = None):
api_key = ""
embed = discord.Embed(title=f"Requested by {ctx.author}")
async with aiohttp.ClientSession() as session:
# search
if search:
embed.description = search
async with session.get(f'http://api.giphy.com/v1/gifs/search?q={search}&api_key={api_key}&limit=10') as response:
data = await response.json()
gif_choice = random.randint(0, 9)
embed.set_image(url=data['data'][gif_choice]['images']['original']['url'])
# radnom
else:
async with session.get(f'http://api.giphy.com/v1/gifs/random?api_key={api_key}&limit=10') as response:
data = await response.json()
embed.set_image(url=data['data']['images']['original']['url'])
await ctx.send(embed=embed)
I'm writing an asyncio script to retrieve stock bars data from Interactive Brokers via the ib_insync library.
While I have the script working, the performance is similar to a serial script. I was hoping to see a drastic improvement in speed. This code will be used in production.
I am new to asyncio and feel like I'm missing an important element. Below is the full script. Would very much appriciate assistance in speeding this up. Thanks.
import asyncio
import ib_insync as ibi
import nest_asyncio
import pandas as pd
nest_asyncio.apply()
class App:
async def run(self, symbols):
print(f"1 start run: {symbols}")
self.ib = ibi.IB()
with await self.ib.connectAsync("127.0.0.1", "****", clientId="****"):
contracts = [ibi.Stock(symbol, "SMART", "USD") for symbol in symbols]
bars_dict = dict()
print(f"2 start loop: {symbols}")
for contract in contracts:
bars = await self.ib.reqHistoricalDataAsync(
contract,
endDateTime="",
durationStr="1 M",
barSizeSetting="1 day",
whatToShow="ADJUSTED_LAST",
useRTH=True,
)
# Convert to dataframes.
bars_dict[contract.symbol] = ibi.util.df(bars)
print(f"3 End bars: {symbols}")
return bars_dict
async def main(self):
res = await asyncio.gather(self.run(self.sp500(0, 100)))
return res
def stop(self):
self.ib.disconnect()
def sp500(self, start=None, end=10):
payload = pd.read_html(
"https://en.wikipedia.org/wiki/List_of_S%26P_500_companies"
)
first_table = payload[0]
sp500 = first_table["Symbol"].sort_values().to_list()
return sp500[start:end]
if __name__ == "__main__":
import time
start = time.time()
app = App()
try:
print(f"START CALL")
res = asyncio.run(app.main())
print(f"END CALL")
except (KeyboardInterrupt, SystemExit):
app.stop()
for ticker, bars in res[0].items():
print(f"{ticker}\n{bars}")
print(f"Total time: {(time.time() - start)}")
Your script is running in sequence. The call to asyncio.gather() in main is useless because it is invoked with just one coroutine. You're supposed to call it with multiple coroutines to have them run in parallel.
For example, you could remove the asyncio.gather() from main (just await self.run(self.sp500(0, 100) there) and instead use it to parallelize calls to reqHistoricalDataAsync:
class App:
async def run(self, symbols):
print(f"1 start run: {symbols}")
self.ib = ibi.IB()
with await self.ib.connectAsync("127.0.0.1", "****", clientId="****"):
contracts = [ibi.Stock(symbol, "SMART", "USD") for symbol in symbols]
print(f"2 start loop: {symbols}")
all_bars = await asyncio.gather(*[
self.ib.reqHistoricalDataAsync(
contract,
endDateTime="",
durationStr="1 M",
barSizeSetting="1 day",
whatToShow="ADJUSTED_LAST",
useRTH=True,
)
for contract in contracts
])
bars_dict = {}
for contract, bars in zip(contracts, all_bars):
# Convert to dataframes.
bars_dict[contract.symbol] = ibi.util.df(bars)
print(f"3 End bars: {symbols}")
return bars_dict
I'm rather new to asyncio and I read through some documentation, examples, and other questions, but I can't find any answers as to what I'm doing wrong here. I don't quite understand enough about asyncio to diagnose why this is happening. I have a class with asyncio functions, one of which opens a websocket and then creates three asyncio tasks which it executes. The first executes completely fine. I'm confused with the execution of the second and third tasks. Even after the second one has executed asyncio.sleep, the third task seems to not execute websock.recv(), and so the second task executes websocket.recv() and gets sent the data that I meant for the third task to receive. What am I doing wrong here? Thanks in advance for any help.
Edit: Sorry the code isn't runnable unless you create or have a discord bot and use it's token in the code.
import websockets
import requests
import asyncio
import json
class GatewayConnection:
def __init__(self):
self.heartbeat = None
self.s = None
self.info = None
self.connected = False
self.uri = f"{self.get_gateway()}/?v=8&encoding=json"
#staticmethod
def get_gateway():
endpoint = requests.get(OAUTH+'gateway')
return endpoint.json()['url']
async def get_gateway_info(self, websock):
recv = json.loads(await websock.recv())
self.heartbeat = recv['d']['heartbeat_interval']
self.s = recv['s']
async def finish_gateway_connect(self, websock):
await websock.send(json.dumps({
'op': 2,
'd': {
'token': TOKEN,
'intents': 513,
'properties': {
'$os': 'windows',
'$browser': 'Sheepp',
'$device': 'Sheepp'
}
}
}))
print("waiting to received ready info")
self.info = json.loads(await websock.recv())
print("Ready info received")
print(self.info)
async def communicate(self):
async with websockets.connect(self.uri) as websock:
self.connected = True
get = asyncio.create_task(self.get_gateway_info(websock))
heartbeat = asyncio.create_task(self.send_heartbeat(websock))
finish = asyncio.create_task(self.finish_gateway_connect(websock))
await get # First task
await heartbeat # Second task
await finish # Third task
async def send_heartbeat(self, websock):
while self.connected:
await websock.send(json.dumps({'op': 1, 'd': self.s}))
print("Sent heartbeat")
response = json.loads(await websock.recv())
print("Received 'heartbeat'")
if response['op'] != 11:
self.connected = False # Discord api says to terminate connection upon not receiving a valid heartbeat response
print(f"The server did not send back a valid heartbeat response, but instead: {response}")
await asyncio.sleep(self.heartbeat/1000)
gateway = GatewayConnection()
try:
loop.run_until_complete(gateway.communicate())
except KeyboardInterrupt:
print("Keyboard Interruption")
finally:
loop.close()
To fix the error I was getting, I made a separate class for handling the websocket which makes sure that the recv and send functions are executed in time correctly. I'm not sure about how well I made the class but here it is:
class WebsocketHandler:
def __init__(self, websocket):
self.ws = websocket
self.sending = False
self.sends = []
self.recvs = []
async def send(self, data):
if self.sending:
self.sends.append(json.dumps(data))
else:
self.sending = True
await self.ws.send(json.dumps(data))
while self.sends:
data = self.sends[0]
self.sends.pop(0)
await self.ws.send(json.dumps(data))
self.sending = False
async def recv(self):
event = asyncio.Event()
self.recvs.append(event)
while self.recvs[0] != event:
await event.wait()
try:
data = await asyncio.wait_for(self.ws.recv(), timeout=1)
except asyncio.exceptions.TimeoutError:
self.recvs.pop(0)
raise asyncio.exceptions.TimeoutError
self.recvs.pop(0)
if len(self.recvs) > 0:
self.recvs[0].set()
return json.loads(data)
The error I was getting was RuntimeError: cannot call recv while another coroutine is already waiting for the next message
Edit: Feel free to critique my code.
Edit 2: I made a couple changes as needed to fix some errors I was getting as well as for convenience sake.
Ok, so I'm attempting to rebuild a GIF bot in Discord.py.
It is based off of some code found on a website that gave a guide on making a discord.py gif bot.
The problem is, the code is old and severely outdated with over 37 errors on it's own.
How would I go about converting this to discord.py-rewrite and basically making it a gif searching bot that can pick up random gifs by tag from GIPHY?
Here is the code I currently have:
import discord.utils
import os
import giphy_client
from giphy_client.rest import ApiException
from pprint import pprint
from discord.ext import commands
import random
from dotenv import load_dotenv
load_dotenv()
API_KEY = os.getenv("GIPHY_API_KEY")
API_TOKEN = os.getenv("GIPHY_API_TOKEN")
api_instance = giphy_client.DefaultApi()
config = {
'api_key': API_KEY,
'token': API_TOKEN,
'limit': 1,
'rating': 'g'
}
try:
api_response = api_instance.gifs_trending_get(
config['token'], limit=config['limit'], rating=config['rating'])
pprint(api_response)
except ApiException as e:
print("Exception when calling DefaultApi->gifs_trending_get: %s\n" % e)
Client = commands.Bot(command_prefix=os.getenv("CLIENT_PREFIX"))
class DiscordClient(discord.Client):
#Client.event
async def on_ready(self):
print("Connected to Discord Client as")
print(f'{self.user.name}#{self.user.discriminator}')
print("-------")
#Client.command(name="gif")
async def search_gifs(query):
try:
giphy_token = API_TOKEN
response = api_instance.gifs_search_get(giphy_token, query, limit=3, rating='g')
lst = list(response.data)
gif = random.choices(lst)
return gif[0].url
except ApiException as e:
return "Exception when calling DefaultApi->gifs_search_get: %s\n" % e
async def on_ready(self):
print("Connected to Discord Client as")
print(Client.user.name)
print("-------")
#Client.command(name='gif')
async def search_gifs(query):
try:
response = api_instance.gifs_search_get(
API_TOKEN, query, limit=3, rating='g')
lst = list(response.data)
gif = random.choices(lst)
return gif[0].url
except ApiException as e:\
return "Exception when calling DefaultApi->gifs_search_get: %s\n" % e\
#Client.command(name='8ball')
async def magic_eight_ball(ctx):
response = [
'Without a doubt.',
'Outlook good.',
'Better not tell you now.',
'Cannot predict now.',
'My reply is no.',
'Outlook not so good.',
]
gif = await search_gifs('cheese')
await ctx.send(random.choice(response))
await ctx.send('Gif URL : ' + gif)
Client.run(os.getenv("CLIENT_TOKEN"))
Here is the website I got the code I'm building off of from:
https://www.maxongzb.com/serving-gifs-with-discord-bot-reading-time-12-mins/
Here is what I currently have for the requirements.txt file:
discord.py
giphy_client
python-dotenv
Any help would be much appreciated.
I host via Heroku due to my VPS taking a dump, so that's why I have the os.getenv("...") lines in there.
I've gotten it down to around 7 errors, 8 warnings and 8 weak warnings. But that's the furthest I was able to get to.