132 lines
5.2 KiB
Python
Executable File
132 lines
5.2 KiB
Python
Executable File
import os
|
|
import sys
|
|
import requests
|
|
import traceback
|
|
import time
|
|
import io
|
|
import binascii
|
|
import logging
|
|
import matplotlib.pyplot as plt
|
|
import numpy as np
|
|
import datetime
|
|
import re
|
|
|
|
rootDirectory = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
sys.path.append(rootDirectory)
|
|
from library.config import glob_the_configs
|
|
from library.prom_api import PromAPI
|
|
|
|
HOSTNAME = None
|
|
|
|
|
|
class SlackSender:
|
|
def __init__(self, token, log):
|
|
self.token = token
|
|
self.alertList = []
|
|
self.log = log
|
|
try:
|
|
self.alertList = glob_the_configs(rootDirectory, \
|
|
rootDirectory + "/AoM_Configs/alert_routing_lookup", \
|
|
'http://consul.service.consul:8500', '127.0.0.1', log)
|
|
except Exception:
|
|
log.error("Failed to load config files: {}".format(traceback.format_exc()))
|
|
|
|
def respond(self, channel, alertId, interval, step):
|
|
self.log.debug("incomming message: channel: {} alert ID: {} interval: {} step: {}".format(channel, alertId, interval, step))
|
|
matchingAlert = next((alert for alert in self.alertList if alert['id'] == alertId), None)
|
|
if not matchingAlert is None:
|
|
query_args = {
|
|
'interval' : matchingAlert['interval'],
|
|
'start_time' : matchingAlert['start_time'],
|
|
'end_time' : matchingAlert['end_time'],
|
|
'query' : matchingAlert['query'],
|
|
}
|
|
prom_api = PromAPI(endpoint=matchingAlert['prometheus_url'])
|
|
if interval :
|
|
try:
|
|
dur = parse_go_duration(interval)
|
|
if dur < 60 :
|
|
dur = 60
|
|
end = 0
|
|
start = -1 * dur
|
|
query_args['start_time'] = start
|
|
query_args['end_time'] = end
|
|
except Exception:
|
|
pass
|
|
if step :
|
|
query_args['interval'] = step
|
|
self.log.debug("QUERY_ARGS {} FROM {} {}".format(query_args, interval, step))
|
|
ret = prom_api.query_range(
|
|
query=query_args['query'],
|
|
start=query_args['start_time'],
|
|
end=query_args['end_time'],
|
|
duration=query_args['interval'])
|
|
if 'status' in ret and ret['status'] == 'success' and 'data' in ret and 'result' in ret['data'] and len(ret['data']['result']) > 0 and 'values' in ret['data']['result'][0] and ret['data']['result'][0]['values'] is not None and len(ret['data']['result'][0]['values']) > 0:
|
|
resultsForGraph = {}
|
|
for row in ret['data']['result'][0]['values']:
|
|
resultsForGraph[row[0]] = row[1]
|
|
finalResults = {}
|
|
for res in resultsForGraph:
|
|
finalResults[time.strftime('%H:%M:%S', time.localtime(res))] = float(resultsForGraph[res])
|
|
plt.clf()
|
|
plt.plot(list(finalResults.keys()), list(finalResults.values()))
|
|
plt.suptitle(alertId + " (all times UTC)")
|
|
if len(finalResults.keys()) > 5:
|
|
tickTuples = [(index, x) for index, x in enumerate(finalResults.keys()) if index % int(len(finalResults.keys()) / 5) == 0]
|
|
tickList = []
|
|
for pair in tickTuples:
|
|
tickList.append(pair[1])
|
|
plt.xticks(ticks = tickList, rotation='vertical')
|
|
else:
|
|
plt.xticks(rotation='vertical')
|
|
plt.ylim(bottom = 0)
|
|
plt.subplots_adjust(bottom=0.2)
|
|
pngData = io.BytesIO()
|
|
fig = plt.gcf()
|
|
fig.savefig(pngData, format = 'png')
|
|
self.sendGraph(channel, pngData)
|
|
else:
|
|
self.log.debug("didn't meet criteria")
|
|
self.sendQueryResults(channel, ret)
|
|
else:
|
|
self.sendQueryResults(channel, "Sorry, I couldn't find a matching alert with ID {}".format(alertId))
|
|
|
|
def sendQueryResults(self, channelId, queryResults):
|
|
response = requests.post('https://slack.com/api/chat.postMessage',
|
|
headers = {
|
|
'Authorization': "Bearer " + self.token,
|
|
'Content-Type': 'application/json; charset=utf-8'
|
|
},
|
|
json = { 'text': queryResults, 'channel': channelId }
|
|
)
|
|
self.log.debug("slack response: {}".format(response.text))
|
|
|
|
def sendGraph(self, channelId, rawData):
|
|
request = requests.Request('POST', 'https://slack.com/api/files.upload',
|
|
data = { 'token': self.token, 'filetype': 'png', 'channels': channelId },
|
|
files = { 'file': ('graph.png', rawData.getvalue(), 'image/png')}
|
|
).prepare()
|
|
self.log.debug("headers to send to Slack: {}".format('\r\n'.join('{}: {}'.format(k, v) for k, v in request.headers.items())))
|
|
self.log.debug("body to send to Slack: {}".format(len(request.body)))
|
|
response = requests.Session().send(request)
|
|
self.log.debug("slack response: {}".format(response.text))
|
|
|
|
def setAlertList(self, newAlertList):
|
|
# TODO can rework to be a dictionary for faster lookup if necessary
|
|
self.alertList = newAlertList
|
|
|
|
def parse_go_duration(duration) :
|
|
duration = str(duration)
|
|
e = Exception("invalid duration "+duration)
|
|
if not re.match("^[0-9][0-9]*[a-z]$", duration) :
|
|
raise e
|
|
unit = duration[-1:]
|
|
n = int(duration[:-1])
|
|
if unit == "s" :
|
|
return n
|
|
if unit == "m" :
|
|
return 60*n
|
|
if unit == "h" :
|
|
return 60*60*n
|
|
raise e
|