Kubernetes Pods Restart Notification alerts on my team's channel - microsoft-teams

Question

My Pods are running on AKS Cluster. Whenever my pods restarted, I had to get a notification on my team's channel, are there any articles or commands to configure the notification?

Answer 1

For that same, you can use tools or application like botkube : https://www.botkube.io/
Also check the Kubewatch : https://github.com/bitnami-labs/kubewatch
You can also implement the Grafana with the Prometheus and Alert manager for monitoring and getting the alert system. : https://github.com/grafana-operator/grafana-operator
However if you can not looking for any tools or applications you can write down the custom script of python, node or any language you are good with and monitor any pod restart event and send the slack hook event.
Sharing one example python code with check the POD running or crashing and send a notification to slack you can update the logic as per need.
from kubernetes import client, config, watch
import json
import requests
import time
logger = logging.getLogger('k8s_events')
logger.setLevel(logging.DEBUG)
# If running inside pod
#config.load_incluster_config()
# If running locally
config.load_kube_config()
v1 = client.CoreV1Api()
v1ext = client.ExtensionsV1beta1Api()
w = watch.Watch()
mydict={}
webhook_url = '';
while True:
pod_list= v1.list_namespaced_pod("default");
for i in pod_list.items:
for c in i.status.container_statuses:
if(c.ready == True):
if i.metadata.name in mydict:
print("Inside mydict If");
print("Pod updated : ",i.metadata.name);
print("My dict value : ",mydict);
mydict[i.metadata.name]['end_time'] = i.status.conditions[1].last_transition_time;
dt_started = mydict[i.metadata.name]['start_time'].replace(tzinfo=None);
dt_ended = mydict[i.metadata.name]['end_time'].replace(tzinfo=None);
duration = str((dt_ended - dt_started).total_seconds()) + ' Sec';
fields = [{"title": "Status", "value": "READY", "short": False }, {"title": "Pod name", "value": i.metadata.name, "short": False }, {"title": "Duration", "value": duration, "short": False }, {"title": "Service name", "value": c.name, "short": False } ]
if c.name not in ('conversation-auto-close-service-scheduler','admin-service-trail-fllow-up-scheduler','bot-trial-email-scheduler','conversation-service-scheduler','faq-service-scheduler','nlp-service-scheduler','refresh-add-on-scheduler','response-sheet-scheduler'):
text = c.name + " Pod is started";
data = {"text": text, "mrkdwn": True, "attachments" : [{"color": "#FBBC05", "title": "Pod Details", "fields" : fields, "footer": "Manvar", "footer_icon": "https://cdn.test.manvar.com/assets/manvar-icon.png"}, ], }
print("Final data to post: ",data);
response = requests.post(webhook_url, data=json.dumps(data),headers={'Content-Type': 'application/json'});
del mydict[i.metadata.name]
if response.status_code != 200:
raise ValueError('Request to slack returned an error %s, the response is:\n%s' % (response.status_code, response.text));
time.sleep(1);
else:
mydict[i.metadata.name] = {"start_time": i.status.conditions[0].last_transition_time,"end_time": i.status.conditions[1].last_transition_time};

Answer 2

I tried out Botkube but I did not want to publicly expose my cluster endpoint, so I wrote the following script based on the code from #Harsh Manvar. You can connect this to Teams using the Incoming Webhook Teams app from Microsoft.
from kubernetes import client, config
import json
import requests
import time
def monitorNamespace(namespace: str, webhookUrl: str):
v1 = client.CoreV1Api()
pod_list= v1.list_namespaced_pod(namespace);
podsNotRunning = {"Namespace": namespace, "Pods": []}
for pod in pod_list.items:
status = getPodStatus(pod)
if status != "Running":
podsNotRunning["Pods"].append({"Podname": pod.metadata.name, "status": status})
if len(podsNotRunning)>0:
sendAlert(podsNotRunning, webhookUrl)
def sendAlert(podsNotRunning, webhookUrl):
print(podsNotRunning)
response = requests.post(webhookUrl, data=json.dumps(podsNotRunning),headers={'Content-Type': 'application/json'});
if response.status_code != 200:
print('Response error:', response)
def getPodStatus(pod: client.models.v1_pod.V1Pod) -> str:
status = pod.status.phase
containerStatus = pod.status.container_statuses[0]
if containerStatus.started is False or containerStatus.ready is False:
waitingState = containerStatus.state.waiting
if waitingState.message is not None:
status = waitingState.reason
return status
if __name__ == "__main__":
# If running inside pod:
#config.load_incluster_config()
# If running locally:
config.load_kube_config()
webhookUrl = 'http://webhookurl'
namespace='default
interval = 10
while True:
monitorNamespace(namespace, webhookUrl)
time.sleep(interval)