import asyncio
import ib_insync as ibi
import symbol_list
import time
start = time.perf_counter()
stocklist = symbol_list.test
endDateTime = '20190328 09:30:00'
durationStr='1 D'
dataDirectory = './data/tmp'
class App:
async def run(self):
self.ib = ibi.IB()
with await self.ib.connectAsync():
contracts = [
ibi.Stock(symbol, 'SMART', 'USD')
for symbol in ['AAPL', 'TSLA', 'AMD', 'INTC']]
for contract in contracts:
# self.ib.reqMktData(contract)
bars = await self.ib.reqHistoricalDataAsync(contract,
endDateTime=endDateTime, durationStr=durationStr,
barSizeSetting='5 mins', whatToShow='MIDPOINT', useRTH=True)
df = ibi.util.df(bars)
df.to_csv(f"{dataDirectory}/{contract.symbol}.csv")
async for tickers in self.ib.pendingTickersEvent:
for ticker in tickers:
print(ticker)
def stop(self):
self.ib.disconnect()
app = App()
try:
asyncio.run(app.run())
except (KeyboardInterrupt, SystemExit):
app.stop()
endtime = (time.perf_counter() - start)/60
print(f"Process time: {endtime:,.2f} minutes")
Please help. I modified the example code from async-streaming-example. I didn't get any error message, but it just runs without giving me the shell prompt. And this code should only take less than a minute, if it runs properly. Essentially, instead of reqMktData, I want to use reqHistoricalDataAsync to get historical data, asynchronously. I've also looked at async execution with ib_insync, but I wasn't able to get that technique to work, either. Could you show me what I'm doing wrong? I welcome any async solutions. Thank you.
I am very new with aiohttp and asyncio so apologies for my ignorance up front. I am having difficulties with the event loop portion of the documentation and don't think my below code is executing asynchronously. I am trying to take the output of all combinations of two lists via itertools, and POST to XML. A more full blown version is listed here while using the requests module, however that is not ideal as I am needing to POST 1000+ requests potentially at a time. Here is a sample of how it looks now:
import aiohttp
import asyncio
import itertools
skillid = ['7715','7735','7736','7737','7738','7739','7740','7741','7742','7743','7744','7745','7746','7747','7748' ,'7749','7750','7751','7752','7753','7754','7755','7756','7757','7758','7759','7760','7761','7762','7763','7764','7765','7766','7767','7768','7769','7770','7771','7772','7773','7774','7775','7776','7777','7778','7779','7780','7781','7782','7783','7784']
agent= ['5124','5315','5331','5764','6049','6076','6192','6323','6669','7690','7716']
url = 'https://url'
user = 'user'
password = 'pass'
headers = {
'Content-Type': 'application/xml'
}
async def main():
async with aiohttp.ClientSession() as session:
for x in itertools.product(agent,skillid):
payload = "<operation><operationType>update</operationType><refURLs><refURL>/unifiedconfig/config/agent/" + x[0] + "</refURL></refURLs><changeSet><agent><skillGroupsRemoved><skillGroup><refURL>/unifiedconfig/config/skillgroup/" + x[1] + "</refURL></skillGroup></skillGroupsRemoved></agent></changeSet></operation>"
async with session.post(url,auth=aiohttp.BasicAuth(user, password), data=payload,headers=headers) as resp:
print(resp.status)
print(await resp.text())
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
I see that coroutines can be used but not sure that applies as there is only a single task to execute. Any clarification is appreciated.
Because you're making a request and then immediately await-ing on it, you are only making one request at a time. If you want to parallelize everything, you need to separate making the request from waiting for the response, and you need to use something like asyncio.gather to wait for the requests in bulk.
In the following example, I've modified your code to connect to a local httpbin instance for testing; I'm making requests to the /delay/<value> endpoint so that each requests takes a random amount of time to complete.
The theory of operation here is:
Move the request code into the asynchronous one_request function,
which we use to build an array of tasks.
Use asyncio.gather to run all the tasks at once.
The one_request functions returns a (agent, skillid, response)
tuple, so that when we iterate over the responses we can tell which
combination of parameters resulted in the given response.
import aiohttp
import asyncio
import itertools
import random
skillid = [
"7715", "7735", "7736", "7737", "7738", "7739", "7740", "7741", "7742",
"7743", "7744", "7745", "7746", "7747", "7748", "7749", "7750", "7751",
"7752", "7753", "7754", "7755", "7756", "7757", "7758", "7759", "7760",
"7761", "7762", "7763", "7764", "7765", "7766", "7767", "7768", "7769",
"7770", "7771", "7772", "7773", "7774", "7775", "7776", "7777", "7778",
"7779", "7780", "7781", "7782", "7783", "7784",
]
agent = [
"5124", "5315", "5331", "5764", "6049", "6076", "6192", "6323", "6669",
"7690", "7716",
]
user = 'user'
password = 'pass'
headers = {
'Content-Type': 'application/xml'
}
async def one_request(session, agent, skillid):
# I'm setting `url` here because I want a random parameter for
# reach request. You would probably just set this once globally.
delay = random.randint(0, 10)
url = f'http://localhost:8787/delay/{delay}'
payload = (
"<operation>"
"<operationType>update</operationType>"
"<refURLs>"
f"<refURL>/unifiedconfig/config/agent/{agent}</refURL>"
"</refURLs>"
"<changeSet>"
"<agent>"
"<skillGroupsRemoved><skillGroup>"
f"<refURL>/unifiedconfig/config/skillgroup/{skillid}</refURL>"
"</skillGroup></skillGroupsRemoved>"
"</agent>"
"</changeSet>"
"</operation>"
)
# This shows when the task actually executes.
print('req', agent, skillid)
async with session.post(
url, auth=aiohttp.BasicAuth(user, password),
data=payload, headers=headers) as resp:
return (agent, skillid, await resp.text())
async def main():
tasks = []
async with aiohttp.ClientSession() as session:
# Add tasks to the `tasks` array
for x in itertools.product(agent, skillid):
task = asyncio.ensure_future(one_request(session, x[0], x[1]))
tasks.append(task)
print(f'making {len(tasks)} requests')
# Run all the tasks and wait for them to complete. Return
# values will end up in the `responses` list.
responses = await asyncio.gather(*tasks)
# Just print everything out.
print(responses)
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
The above code results in about 561 requests, and runs in about 30
seconds with the random delay I've introduced.
This code runs all the requests at once. If you wanted to limit the
maximum number of concurrent requests, you could introduce a
Semaphore to make one_request block if there were too many active requests.
If you wanted to process responses as they arrived, rather than
waiting for everything to complete, you could investigate the
asyncio.wait method instead.
Im trying to use boto3 in a job of AWS Glue to call a Lambda Function but without results.
I upload a zip with the libraries:
Like the examples by AWS
and without a zip.
The error is this " Unable to load data for: endpoints".
Im trying to invoke without zip but this go to a timeout exception.
import boto3
client = boto3.client('lambda' , region_name='us-east-1')
r_lambda = client.invoke(FunctionName='S3GlueJson')
Can someone help me ?
In Python, use Boto3 Lambda client 'invoke()'. For example, you can create a Lambda container, then call that from a Glue Job:
import boto3
import pandas as pd
lambda_client = boto3.client('lambda',region_name='us-east-1')
def get_predictions( df ):
# Call getPredictions Lambda container
response = lambda_client.invoke(
FunctionName='getPredictions',
InvocationType='RequestResponse',
LogType='Tail',
Payload=df
)
logger.info('Received response from Lambda container.')
data = response["Payload"].read().decode('utf-8')
x = json.loads(data)
df_pred = pd.DataFrame.from_dict(x)
return df_pred
dfjson = df.to_json()
df_pred = get_predictions( dfjson )
df_pred.head()
If you want to call a Glue Jobs from Lambda Function, can do it like this:
import boto3
glue = boto3.client(service_name='glue', region_name='us-east-1',
endpoint_url='https://glue.us-east-1.amazonaws.com')
#Start Job
myNewJobRun = glue.start_job_run(JobName=JOB_NAME)
#Get current state of Job, to be sure it's running
status = glue.get_job_run(JobName=JOB_NAME, RunId=myNewJobRun['JobRunId'])
logger.info('JOB State {}: {}'.format(
JOB_NAME, status['JobRun']['JobRunState']))
As Job execution can late some time to finish, it's better to don't wait on Lambda function for it to finish.
I have a Gevent based backend, where in a set of external apis are called in parallel. NewRelic doesn't report the time consumed by these apis in its UI.
I tried to push these timings myself using custom metrics. Here is the code snippet:
import time
t1 = time.time()
instance.getAval()
newrelic.agent.record_custom_metric("Custom/BUS_EXT_CALL", time.time() - t1)
This didn't work either. I tried to look into their python agent code and made following changes:
def record_custom_metric(name, value, application=None, app_name=None):
if app_name is None and application is None:
transaction = current_transaction()
if transaction:
transaction.record_custom_metric(name, value)
elif app_name is not None:
application = newrelic.api.application.application_instance(app_name)
application.record_custom_metric(name, value)
else:
if application.enabled:
application.record_custom_metric(name, value)
and now newrelic.agent.record_custom_metric("Custom/BUS_EXT_CALL", time.time() - t1, app_name="ControllerTest") this call was successful, but somehow the data doesn't appear in NewRelic's dashboard.
I would like to know if is there an option to send a notification when log time worked is reaching estimating time.
example:
Start.
Estimated:
Original Estimate - 5 minutes
5m
Remaining:
Remaining Estimate - 5 minutes
5m
Logged:
Time Spent - Not Specified
Not Specified
When i Log time.
Estimated:
Original Estimate - 5 minutes
5m
Remaining:
Time Spent - 4 minutes Remaining Estimate - 1 minute
1m
Logged:
Time Spent - 4 minutes Remaining Estimate - 1 minute
4m
I like JIRA send the notification before 1 minute ending or what ever i set.
I'm sorry for the bad english.
Thank you
I believe you wanted to create some kind of triggered event when logged work approaches original estimate, but I do not know how you could do that in JIRA. Nevertheless, I know something that still might help you to solve your problem.
Try using following groovy script:
import com.atlassian.jira.ComponentManager
import com.atlassian.jira.component.ComponentAccessor
import com.atlassian.jira.config.properties.APKeys
import com.atlassian.jira.config.properties.ApplicationProperties
import com.atlassian.jira.issue.Issue
import com.atlassian.jira.issue.search.SearchResults
import com.atlassian.jira.issue.worklog.Worklog
import com.atlassian.jira.jql.parser.DefaultJqlQueryParser
import com.atlassian.jira.web.bean.PagerFilter
import com.atlassian.mail.Email
import com.atlassian.mail.MailException
import com.atlassian.mail.MailFactory
import com.atlassian.mail.queue.SingleMailQueueItem
import com.atlassian.query.Query
import groovy.text.GStringTemplateEngine
import org.apache.log4j.Logger
import com.atlassian.core.util.DateUtils
def componentManager = ComponentManager.getInstance()
def worklogManager = componentManager.getWorklogManager()
def userUtil = componentManager.getUserUtil()
def user = userUtil.getUser('admin')
def searchProvider = componentManager.getSearchProvider()
def queryParser = new DefaultJqlQueryParser()
Logger log = Logger.getLogger('worklogNotification')
Query jql = queryParser.parseQuery('project = ABC and updated > startOfDay(-1d)')
SearchResults results = searchProvider.search(jql, user, PagerFilter.getUnlimitedFilter())
List issues = results.getIssues()
String emailFormat = 'HTML'
def mailServerManager = componentManager.getMailServerManager()
def mailServer = mailServerManager.getDefaultSMTPMailServer()
String defaultSubject = 'Logged work on JIRA issue %ISSUE% exceeds original estimate'
String body = ''
Map binding = [:]
String loggedWorkDiff = ''
String template = '''
Dear ${issue.assignee.displayName}, <br /><br />
Logged work on issue ${issue.key} (${issue.summary}) exceeds original estimate ($loggedWorkDiff more than expected).<br />
*** This is an automatically generated email, you do not need to reply ***<br />
'''
GStringTemplateEngine engine = new GStringTemplateEngine()
ApplicationProperties applicationProperties = componentManager.getApplicationProperties()
binding.put("baseUrl", applicationProperties.getString(APKeys.JIRA_BASEURL))
if (mailServer && ! MailFactory.isSendingDisabled()) {
for (Issue issue in issues) {
if(issue.originalEstimate) {
loggedWork = 0
worklogs = worklogManager.getByIssue(issue)
worklogs.each{Worklog worklog -> loggedWork += worklog.getTimeSpent()}
if(loggedWork - issue.originalEstimate) {
loggedWorkDiff = DateUtils.getDurationString(Math.round(loggedWork - issue.originalEstimate))
email = new Email(issue.getAssigneeUser().getEmailAddress())
email.setFrom(mailServer.getDefaultFrom())
email.setSubject(defaultSubject.replace('%ISSUE%', issue.getKey()))
email.setMimeType(emailFormat == "HTML" ? "text/html" : "text/plain")
binding.put("issue", issue)
binding.put('loggedWorkDiff', loggedWorkDiff)
body = engine.createTemplate(template).make(binding).toString()
email.setBody(body)
try {
log.debug ("Sending mail to ${email.getTo()}")
log.debug ("with body ${email.getBody()}")
log.debug ("from template ${template}")
SingleMailQueueItem item = new SingleMailQueueItem(email);
ComponentAccessor.getMailQueue().addItem(item);
}
catch (MailException e) {
log.warn ("Error sending email", e)
}
}
}
}
}
This script takes issues for project ABC, which have been updated during the previous day (JQL: "project = ABC and updated > startOfDay(-1d)"), calculates difference between logged work and estimated work and sends notification to the issue assignee if logged work exceeds original estimate.
You can add this script to the list of JIRA services (JIRA -> Administration -> System -> Advanced -> Services).
Name: [any name]
Class: com.onresolve.jira.groovy.GroovyService
Delay: 1440
Input file: [path to your script on server]
Delay: 1440
Note that you will put 1440 (min) as a service delay, which equals to one day. So, the script will be executed once per day sending notification to issue assignees about exceeded original estimates.
Also note that Groovy Scripting plugin should be installed in order to be able to run your script.