import os, sys, json, base64, re; from kubernetes import client, config def handler(event, context): if 'data' in event: message = json.loads(base64.b64decode(event['data']).decode('utf-8')) print('Data found - for debugging purposes, message contents printed in next line') print(message) else: print('Data not found - this is an unexpected error, and the function has been called without a Pub/Sub message. This should be checked') return f'Data not found' if 'pod_name' in message['incident']['resource']['labels']: namespace = message["incident"]['resource']['labels']['namespace_name'] pod = message["incident"]['resource']['labels']['pod_name'] cluster = message['incident']['resource']['labels']['cluster_name'] print("Deleting " + pod + " in namespace " + namespace + " in cluster " + cluster + " because it has utilised more than 90% of the configured memory limit") elif "deadlock_detected" in message['incident']['metric']['type']: namespace = namespace = message["incident"]['resource']['labels']['namespace_name'] sum = message['incident']['summary'] extract = sum.split("pod_name=", 1)[-1] pod = re.findall("[\w-]*", extract)[0] cluster = message['incident']['resource']['labels']['cluster_name'] print("Deleting " + pod + " in namespace " + namespace + " in cluster " + cluster + " because it has encountered a deadlock condition") elif "request-timeout" in message['incident']['metric']['type']: namespace = namespace = message["incident"]['resource']['labels']['namespace_name'] sum = message['incident']['summary'] extract = sum.split("pod_name=", 1)[-1] pod = re.findall("[\w-]*", extract)[0] cluster = message['incident']['resource']['labels']['cluster_name'] print("Deleting " + pod + " in namespace " + namespace + " in cluster " + cluster + " because it has encountered a frequent request-timeout condition") else: print('From unknown source') if 'KUBERNETES_LOAD_KUBE_CONFIG' in os.environ: if cluster == 'ggv-k8s-prod': config.load_kube_config(context='gke_api-project-1004752207173_us-east4_ggv-k8s-prod', persist_config=False) if cluster == 'ggv-k8s-cluster': config.load_kube_config(context='gke_api-project-1004752207173_asia-southeast1_ggv-k8s-cluster', persist_config=False) if cluster == 'ggv-k8s-production-sg': config.load_kube_config(context='gke_api-project-1004752207173_asia-southeast1_ggv-k8s-production-sg', persist_config=False) else: config.load_incluster_config() v1 = client.CoreV1Api() body = client.V1beta1Eviction(metadata=client.V1ObjectMeta(name=pod, namespace=namespace)) api_response = v1.create_namespaced_pod_eviction(name=pod, namespace=namespace, body=body) print('Evicted pod?')