46 lines
2.9 KiB
Python
46 lines
2.9 KiB
Python
import os, sys, json, base64, re;
|
|
from kubernetes import client, config
|
|
|
|
def handler(event, context):
|
|
if 'data' in event:
|
|
message = json.loads(base64.b64decode(event['data']).decode('utf-8'))
|
|
print('Data found - for debugging purposes, message contents printed in next line')
|
|
print(message)
|
|
else:
|
|
print('Data not found - this is an unexpected error, and the function has been called without a Pub/Sub message. This should be checked')
|
|
return f'Data not found'
|
|
if 'pod_name' in message['incident']['resource']['labels']:
|
|
namespace = message["incident"]['resource']['labels']['namespace_name']
|
|
pod = message["incident"]['resource']['labels']['pod_name']
|
|
cluster = message['incident']['resource']['labels']['cluster_name']
|
|
print("Deleting " + pod + " in namespace " + namespace + " in cluster " + cluster + " because it has utilised more than 90% of the configured memory limit")
|
|
elif "deadlock_detected" in message['incident']['metric']['type']:
|
|
namespace = namespace = message["incident"]['resource']['labels']['namespace_name']
|
|
sum = message['incident']['summary']
|
|
extract = sum.split("pod_name=", 1)[-1]
|
|
pod = re.findall("[\w-]*", extract)[0]
|
|
cluster = message['incident']['resource']['labels']['cluster_name']
|
|
print("Deleting " + pod + " in namespace " + namespace + " in cluster " + cluster + " because it has encountered a deadlock condition")
|
|
elif "request-timeout" in message['incident']['metric']['type']:
|
|
namespace = namespace = message["incident"]['resource']['labels']['namespace_name']
|
|
sum = message['incident']['summary']
|
|
extract = sum.split("pod_name=", 1)[-1]
|
|
pod = re.findall("[\w-]*", extract)[0]
|
|
cluster = message['incident']['resource']['labels']['cluster_name']
|
|
print("Deleting " + pod + " in namespace " + namespace + " in cluster " + cluster + " because it has encountered a frequent request-timeout condition")
|
|
else:
|
|
print('From unknown source')
|
|
|
|
if 'KUBERNETES_LOAD_KUBE_CONFIG' in os.environ:
|
|
if cluster == 'ggv-k8s-prod':
|
|
config.load_kube_config(context='gke_api-project-1004752207173_us-east4_ggv-k8s-prod', persist_config=False)
|
|
if cluster == 'ggv-k8s-cluster':
|
|
config.load_kube_config(context='gke_api-project-1004752207173_asia-southeast1_ggv-k8s-cluster', persist_config=False)
|
|
if cluster == 'ggv-k8s-production-sg':
|
|
config.load_kube_config(context='gke_api-project-1004752207173_asia-southeast1_ggv-k8s-production-sg', persist_config=False)
|
|
else:
|
|
config.load_incluster_config()
|
|
v1 = client.CoreV1Api()
|
|
body = client.V1beta1Eviction(metadata=client.V1ObjectMeta(name=pod, namespace=namespace))
|
|
api_response = v1.create_namespaced_pod_eviction(name=pod, namespace=namespace, body=body)
|
|
print('Evicted pod?') |