Error: Runtime exited without providing a reason in python lambda - aws-lambda

This code is used to do following
This lambda gets triggered via an event rule
Event is sent when and instance state goes to running/terminated
When instance is running the instance attributes are saved to dynamodb table
Route53 record sets are created / deleted when instance is in running/ terminated state.
Now the lambda created records when instance is launched and throws below error
RequestId: 9f4fb9ed-88db-442a-bc4f-079744f5bbcf Error: Runtime exited without providing a reason
Runtime.ExitError
import ipaddress
import os
import time
from datetime import datetime
from typing import Dict, List, Optional
import boto3
from botocore.exceptions import ClientError, ParamValidationError
from pynamodb.attributes import UnicodeAttribute, UTCDateTimeAttribute
from pynamodb.exceptions import DoesNotExist
from pynamodb.models import Model
def lambda_handler(event, context):
"""Registers or de-registers private DNS resource records for a given EC2 instance."""
# Retrieve details from invocation event object.
try:
account_id = event["account"]
instance_id = event["detail"]["instance-id"]
instance_region = event["region"]
instance_state = event["detail"]["state"]
except KeyError as err:
raise RuntimeError(
f"One or more required fields missing from event object {err}"
)
print(
f"EC2 instance {instance_id} changed to state `{instance_state}` in account "
f"{account_id} and region {instance_region}."
)
print(f"Creating a new aws session in {instance_region} for account {account_id}.")
target_session = aws_session(
region=instance_region, account_id=account_id, assume_role=ASSUME_ROLE_NAME,
)
print(f"Retrieving instance and VPC attributes for instance {instance_id}.")
instance_resource = get_instance_resource(instance_id, target_session)
vpc_resource = get_vpc_resource(instance_resource.vpc_id, target_session)
route53_client = target_session.client("route53")
print(f"Retrieving DNS configuration from VPC {instance_resource.vpc_id}.")
forward_zone = get_vpc_domain(vpc_resource)
print(f"Calculating reverse DNS configuration for instance {instance_id}.")
reverse_lookup = determine_reverse_lookup(instance_resource, vpc_resource)
if instance_state == "running":
print(f"Building DNS registration record for instance {instance_id}.")
#vpc_resource = get_vpc_resource(instance_resource.vpc_id, target_session)
#print(f"Retrieving DNS configuration from VPC {instance_resource.vpc_id}.")
#forward_zone = get_vpc_domain(vpc_resource)
#print(f"Calculating reverse DNS configuration for instance {instance_id}.")
#reverse_lookup = determine_reverse_lookup(instance_resource, vpc_resource)
record = Registration(
account_id=account_id,
hostname=generate_hostname(instance_resource),
instance_id=instance_resource.id,
forward_zone=forward_zone,
forward_zone_id=get_zone_id(forward_zone, route53_client),
private_address=instance_resource.private_ip_address,
region=instance_region,
reverse_hostname=reverse_lookup["Host"],
reverse_zone=reverse_lookup["Zone"],
reverse_zone_id=get_zone_id(reverse_lookup["Zone"], route53_client),
vpc_id=instance_resource.vpc_id,
)
print(record)
try:
if record.forward_zone_id is not None:
manage_resource_record(record, route53_client)
if record.forward_zone_id and record.reverse_zone_id is not None:
manage_resource_record(record, route53_client, record_type="PTR")
except RuntimeError as err:
print(f"An error occurred while creating records: {err}")
exit(os.EX_IOERR)
if record.forward_zone_id:
print(
f"Saving DNS registration record to database for instance {instance_id}."
)
record.save()
else:
print(
f"No matching hosted zone for {record.forward_zone} associated "
f"with {record.vpc_id}."
)
else:
try:
print(
f"Getting DNS registration record from database for instance {instance_id}."
)
record = Registration.get(instance_id)
if record.forward_zone_id is not None:
manage_resource_record(record, route53_client, action="DELETE")
if record.reverse_zone_id is not None:
manage_resource_record(record, route53_client, record_type="PTR", action="DELETE")
print(
"Deleting DNS registration record from database for "
f"instance {instance_id}."
)
record.delete()
except DoesNotExist:
print(f"A registration record for instance {instance_id} does not exist.")
exit(os.EX_DATAERR)
except RuntimeError as err:
print(f"An error occurred while removing resource records: {err}")
exit(os.EX_IOERR)
exit(os.EX_OK)

I had this problem with a dotnet lambda. Turns out it'd run out of memory. Raising the memory ceiling allowed it to pass.

exit(os.EX_OK) statement in last line was causing this. Removing this line resolved my issue.

What I did to get around this, but still have an exit code if there was errors detected was the following:
def main(event=None, context=None):
logger.info('Starting Lambda')
error_count = 0
s3_objects = parse_event(event)
for s3_object in s3_objects:
logger.info('Parsing s3://{}/{}'.format(s3_object['bucket'], s3_object['key']))
error_count += parse_object(object_json)
logger.info('Total Errors: {}'.format(error_count))
if error_count > 255:
error_count = 255
logger.info('Exiting lambda')
# exit if error_count (lambda doesn't like sys.exit(0))
if error_count > 0:
sys.exit(error_count)
TL;DR - Exit only if an error is detected.

Related

Turn on All ec2(+ future created ec2) 'termination protection' Using Lambda

Im trying to turn on 'termination protection' for all ec2.
(termination protection doesn't work to spot instance, so i want to add skip condition not to make an error for spot instance.)
I saw a code like below, however the code doesn't work.
import json
import boto3
def lambda_handler(event, context):
client = boto3.client('ec2')
ec2_regions = [region['RegionName'] for region in client.describe_regions()['Regions']]
for region in ec2_regions:
client = boto3.client('ec2', region_name=region)
conn = boto3.resource('ec2',region_name=region)
instances = conn.instances.filter()
for instance in instances:
if instance.state["Name"] == "running":
#print instance.id # , instance.instance_type, region)
terminate_protection=client.describe_instance_attribute(InstanceId =instance.id,Attribute = 'disableApiTermination')
protection_value=(terminate_protection['DisableApiTermination']['Value'])
if protection_value == False:
client.modify_instance_attribute(InstanceId=instance.id,Attribute="disableApiTermination",Value= "True" )
Summary,,,
I want to turn on 'Termination protection' for all EC2 which is running(not spot instance).
The region should be ap-northeast-2.
Could you help me to fix this code to running appropriatly?
if you want to skip the spot instance all you need to do this is figure out which one is spot instance.
You need to use describe_instances api and then using if-else condition, request_id is empty its a spot instance, if not then its not a spot instance
import boto3
ec2 = boto3.resource('ec2')
instances = ec2.instances.filter(Filters=[{'Name': 'instance-state-name', 'Values': ['running']}]) #add filter of your own choice
for instance in instances:
if instance.spot_instance_request_id:
# logic to skip termination ( spot instance )
else:
# logic to terminate ( not spot instance )
You can refer a similar question on this -> https://stackoverflow.com/a/45604396/13126651
docs for describe_instances

ClowdWatch doesn't show any AWS lambda failure details

I'm trying to debug my lambda_function.py in AWS.
It writes the logs to CloudWatch always but..
In some case (cannot understand which) of 'Internal Server Error' it doesnt write anything but only START and END records to CloudWatch, which makes impossible to understand the root cause of the failure.
Here is my code:
import json
import psycopg2
def lambda_handler(event, context):
try:
print('started')
s = psycopg2.__version__
print(s)
conn = psycopg2.connect(
user='pg_user',
password='*********',
host='pg_host',
port='5432',
database='dev_db'
)
cur = conn.cursor()
cur.execute("select count(1) q from keywords_to_scrape")
for q in cur:
print(f'q = {q}')
except Exception as e:
print(f'exception: {e} ')
finally:
print('returning result')
return {
'statusCode' : 200,
'body' : json.dumps(f'{s}')
}
and if to comment this part
.............
#conn = psycopg2.connect(
# user='pg_user',
# password='*********',
# host='pg_host',
# port='5432',
# database='dev_db'
#)
.............
then it perfectly writes to CloudWatch the lines "started", "exception" with clear exception message and finally returns 200 OK
But with the lines of connection to DB it just dies with 'Internal server error' and with no messages in CloudWatch.
Could you please advice how to track such failures?
You are hitting timeout error as according to your comment.
Task timed out after 3.01 seconds
A few things for you to try and check:
Make your Lambda Timeout longer. E.g. 10 seconds.
If your Lambda is still hitting timeout error after you longer your Lambda Timeout, then you might want to check your database connections to the database. E.g. Make sure your Lambda is placed in the same VPC as your database and your database security group enables traffic from your Lambda.

How do I get rid of the Runtime.MarshalError in SQS URL

Trying to add some basic error handling & logging to a working Lambda function. After adding the elements I now receive this error:
{
"errorMessage": "Unable to marshal response: sqs.Queue(url='https://us-west-2.queue.amazonaws.com/225565555556/Messages') is not JSON serializable",
"errorType": "Runtime.MarshalError",
"stackTrace": []
}
My searches on Stack Overflow have led me to believe something needs to be converted to a string object but I don't know where to fix that. Here is the entire function:
# Backport print_function for backwards compatibility
from __future__ import print_function
import logging
# Use built-in package for encoding/decoding JSON data
import json
# Module required to work with Boto3 environment variables
import os
# Module provides classes for manipulating date/time
from datetime import datetime
# AWS Python SDK module
import boto3
from botocore.exceptions import ClientError
# Reference function environment variables
QUEUE_NAME = os.environ['QUEUE_NAME']
MAX_QUEUE_MESSAGES = os.environ['MAX_QUEUE_MESSAGES']
DYNAMODB_TABLE = os.environ['DYNAMODB_TABLE']
# Create AWS service resource objects
sqs = boto3.resource('sqs')
dynamodb = boto3.resource('dynamodb')
logger = logging.getLogger(__name__)
# Define function entry point
def lambda_handler(event, context):
# Use service resource to call API to retrieve SQS queue name
try:
queue = sqs.get_queue_by_name(QueueName=QUEUE_NAME)
logger.info("Got queue '%s' with URL=%s", QUEUE_NAME, queue.url)
except ClientError as error:
logger.exception("Couldn't get queue named %s.", QUEUE_NAME)
raise error
else:
return queue
# Print the number of messages waiting in queue for consumer
print("ApproximateNumberOfMessages:",
queue.attributes.get('ApproximateNumberOfMessages'))
# Iterate through message event records
for message in event['Records']:
print("Starting your Lambda Function...")
body = message["body"]
id = message['messageId']
print(str(body))
# Write message to DynamoDB
table = dynamodb.Table(DYNAMODB_TABLE)
# Call DDB API to add message item to table variable
response = table.put_item(
Item={
'MessageId':message['messageId'],
'Body':message['body'],
'Timestamp':datetime.now().isoformat()
},
)
print("Wrote message to DynamoDB:", json.dumps(response))
The resolution to this error was to change the else statement from:
else:
return queue
To:
else:
return str(queue)

Is it alright to inlcude connect() inside the lambda_handler in order to close the connection after use?

I wrote one lambda function to access the MySQL database and fetch the data i.e to fetch the number of users, but any real-time update is not fetched, unless the connection is re-established.
And closing the connection inside the lambda_handler before returning, results in connection error upon its next call.
The query which I am using is -> select count(*) from users
import os
import pymysql
import json
import logging
endpoint = os.environ.get('DBMS_endpoint')
username = os.environ.get('DBMS_username')
password = os.environ.get('DBMS_password')
database_name = os.environ.get('DBMS_name')
DBport = int(os.environ.get('DBMS_port'))
logger = logging.getLogger()
logger.setLevel(logging.INFO)
try:
connection = pymysql.connect(endpoint, user=username, passwd=password, db=database_name, port=DBport)
logger.info("SUCCESS: Connection to RDS mysql instance succeeded")
except:
logger.error("ERROR: Unexpected error: Could not connect to MySql instance.")
def lambda_handler(event, context):
try:
cursor = connection.cursor()
............some.work..........
............work.saved..........
cursor.close()
connection.close()
return .....
except:
print("ERROR")
The above code results in connection error after its second time usage,
First time it works fine and gives the output but the second time when I run the lambda function it results in connection error.
Upon removal of this line ->
connection.close()
The code works fine but the real-time data which was inserted into the DB is not fetched by the lambda,
but when I don't use the lambda function for 2 minutes, then after using it again, the new value is fetched by it.
So,
In order to rectify this problem,
I placed the connect() inside the lambda_handler and the problem is solved and it also fetches the real-time data upon insertion.
import os
import pymysql
import json
import logging
endpoint = os.environ.get('DBMS_endpoint')
username = os.environ.get('DBMS_username')
password = os.environ.get('DBMS_password')
database_name = os.environ.get('DBMS_name')
DBport = int(os.environ.get('DBMS_port'))
logger = logging.getLogger()
logger.setLevel(logging.INFO)
def lambda_handler(event, context):
try:
try:
connection = pymysql.connect(endpoint, user=username, passwd=password, db=database_name, port=DBport)
except:
logger.error("ERROR: Unexpected error: Could not connect to MySql instance.")
cursor = connection.cursor()
............some.work..........
............work.saved..........
cursor.close()
connection.close()
return .....
except:
print("ERROR")
So, I want to know, whether is it right to do this, or there is some other way to solve this problem, I trying to solve this for few-days and finally this solution is working, but not sure whether will it be a good practice to do this or not.
Any problems will occur if the number of connections to database increases?
Or any kind of resource problem?

How to get email message in outlook account using IMAP?

I tried one sample program for getting an email message in outlook account using IMAP. In this account, I have 20 folders its getting all email messages except these folders (contact, calendar, task) not getting data its throwing server error. How to fix this error.
Code
import imaplib
import pprint
import email
import base64
import json
import re
import os
import fileinput
imap_host = 'outlook.office365.com'
imap_user = 'XXXXXXXXXXX'
imap_pass = 'XXXXXXXXXXXXX'
count = 0
file_path = 'geek.txt'
# connect to host using SSL
imap = imaplib.IMAP4_SSL(imap_host,993)
# login to server
l = imap.login(imap_user, imap_pass)
# Get Flags,mailbox_name,delimiter using regex
list_response_pattern = re.compile(r'\((?P<flags>.*?)\) "(?P<delimiter>.*)" (?P<name>.*)')
# Get List of Sync folders
list_data = imap.list()
# Check Local Storage is empty Sync All Folders Details.
print(os.stat(file_path).st_size)
if os.stat(file_path).st_size == 0:
global day
# Iterate folders in Sync folder
for i in list_data[1]:
# Get Folder name
sample = re.findall('"\/"(.*)',i.decode("utf-8"))
# Get Message_ids
try:
print("message")
print(sample[0].lstrip().strip('"'))
data = imap.select(sample[0].lstrip())
search_resp, search_data = imap.search( None, "ALL" )
match = list_response_pattern.match(i.decode("utf-8"))
flags, delimiter, mailbox_name = match.groups()
print("1")
print(mailbox_name)
mailbox_name = mailbox_name.strip('"')
print(mailbox_name)
except Exception as e:
print(e)
continue
# Get Current Status of Folder
current_status = imap.status(
'"{}"'.format(mailbox_name),
'(MESSAGES RECENT UIDNEXT UIDVALIDITY UNSEEN)',
)
print(current_status)
# Get message using UID and Message_id
msg_ids = search_data[ 0 ].split()
print("total count: ",len(msg_ids))
for i in msg_ids:
print("$$$$$$$$$$$$$$$$$$$$")
print("Message Ids: ", i)
count = count + 1
fetch_resp, fetch_UID = imap.fetch( i, 'UID' )
print("Fetch UID: ", fetch_UID)
day = bytes(str(fetch_UID[0].split()[2]).split("'")[1].split(')')[0],'utf-8')
print("ID: ",day)
fetch_resp, fetch_mdg = imap.uid('fetch', day, '(RFC822)')
print(fetch_mdg)
print("$$$$$$$$$$$$$$$$$$$$$")
email_msg = fetch_mdg[0][1]
if email_msg and isinstance(email_msg, str):
try:
email_msg = email.message_from_string(email_msg)
except :
email_msg = None
elif email_msg and isinstance(email_msg, bytes):
try:
email_msg = email.message_from_bytes(email_msg)
except:
email_msg = None
print("*********************************")
print("Count: ",count)
print("UID: ",day)
print(mailbox_name)
print(email_msg['To'])
print(email_msg['From'])
print(email_msg['subject'])
print(email_msg)
print("*********************************")
# Store Folder details in File
status_details = current_status[1][0].decode("utf-8")
status_details = status_details.split('(')[1].split(')')[0].split(' ')
print(status_details)
if len(msg_ids) == 0:
json1 = json.dumps({'total_count':int(status_details[1]),'UID':0,'UIDNext':int(status_details[5]),'UIDValidity':int(status_details[7]), 'Folder name':mailbox_name})
else:
json1 = json.dumps({'total_count':int(status_details[1]),'UID':int(day),'UIDNext':int(status_details[5]),'UIDValidity':int(status_details[7]), 'Folder name':mailbox_name})
file = open(file_path,'a')
file.write(json1)
file.write("\n")
print('hi')
Response
$$$$$$$$$$$$$$$$$$$$
Message Ids: b'3'
Fetch UID: [b'3 (UID 11)']
ID: b'11'
[(b'3 (RFC822 {757}', b'MIME-Version: 1.0\r\nContent-Type: text/plain; charset="us-ascii"\r\nFrom: Microsoft Exchange Server\r\nTo: "\r\nSubject: Retrieval using the IMAP4 protocol failed for the following message:\r\n 11\r\nContent-Transfer-Encoding: quoted-printable\r\n\r\nThe server couldn\'t retrieve the following message:\r\n\r\nSubject: "Test email Sync 3"\r\nFrom: "Imap Testing" ("/O=3DEXCHANGELABS/OU=3DEXCHANGE ADMINISTRATIVE GROUP=\r\n (FYDIBOHF23SPDLT)/CN=3DRECIPIENTS/CN=3DEBF2483D9A0145A59A48B829B12A45E4-MA=\r\nILBOX1")\r\nSent date: 5/6/2020 2:02:59 AM\r\n\r\nThe message hasn\'t been deleted. You might be able to view it using either =\r\nOutlook or Outlook Web App. You can also contact the sender to find out wha=\r\nt the message says.=\r\n'), b' UID 11 FLAGS (\\Seen))']
$$$$$$$$$$$$$$$$$$$$$
Server Error
Subject: Retrieval using the IMAP4 protocol failed for the following message:
7
Content-Transfer-Encoding: quoted-printable
The server couldn't retrieve the following message:
Subject: "Testing"
Sent date: 5/6/2020 2:01:54 AM
The message hasn't been deleted. You might be able to view it using either =
Outlook or Outlook Web App. You can also contact the sender to find out wha=
t the message says.=
I have around 20 folders I iterate one by one get current status of folder and stored in sample file. Its successfully working.but I tried to print email messages some folders (contact,calender,task) its showing this response.

Resources