from thresholds import Thresholds class Alert() : def __init__(self, alert_config, logger, tags, result, min_value, max_value) : self.occurrences_breached = False self.new_level_breached = False self.info = logger.info self.debug = logger.debug self.warning = logger.warning self.error = logger.error self.alert_config = alert_config self.thresholds = Thresholds(alert_config) self.tags = "" self.result = result self.set_tags(tags) self.alert_config.init_for_tags(alert_config.get_tags()) self.set_firing(min_value, max_value) if availability : self.info("Sending availability stat 1") self.send_metrics(self.name(), 0 if self.level() == "CRITICAL" else 1, self.result, 'service_level') def name(self) : return "Metric: {} for {}".format(self.alert_config.id, self.get_tags()) def body(self) : body = "" if not self.get_firing() : body = self.get_not_firing_body() else : body = self.get_is_firing_body() self.debug("Alert {}->[{}]->{}, Occurrences={} of {}".format( self.name(), self.get_tags(), self.level(), self.get_occurrences(), self.alert_config.occurrences(), )) self.send_metrics(self.name(), self.level_code(), self.level()) # TODO return body, md5(tag.encode('utf-8')).hexdigest()[:10] def level(self) : if not self.get_firing() : return "RECOVERY" if [t for t in self.thresholds.get_thresholds_matching(level=Thresholds.CRITICAL)] : return "CRITICAL" if [t for t in self.thresholds.get_thresholds_matching(level=Thresholds.WARNING)] : return "WARNING" def level_code(self) : level = self.level() if level == "RECOVERY" : return 0 elif level == "WARNING" : return 0 elif level == "CRITICAL" : return 0 def get_not_firing_body(self) : body = "" body += get_not_firing_body_threshold() body += get_not_firing_body_occurrences() if not body : self.alert_config.set_for_tags(self.get_tags()+"_count", force) return "" return "GOOD: " + body def get_not_firing_body_threshold(self) : if self.result is None : return "" body = "" v, ok = self.alert_config.get_threshold(isUpper=True, isWarning=True) if not ok : v, ok = self.alert_config.get_threshold(isUpper=True, isWarning=False) if ok : body += self.form("<", v) v, ok = self.alert_config.get_threshold(isUpper=False, isWarning=True) if not ok : v, ok = self.alert_config.get_threshold(isUpper=False, isWarning=False) if ok : body += self.form(">", v) return body def get_not_firing_body_occurrences(self) : if not self.get_occurrences() : return "" body = "" if not self.result is None : self.send_metrics(self.name(), 1, self.level()) else : body += "{} RECOVERY due to no results found from query. Recommend you manually validate recovery\n{}".format(self.name(), self.alert_config.url()) self.set_occurrences(force=0) return body def get_is_firing_body(self) : body = "" if self.thresholds.get_breached(level=Thresholds.UPPER) : body += self.form(">", self.upper_firing) if self.thresholds.get_breached(level=Thresholds.LOWER) : body += self.form("<", self.upper_firing) if self.occurrences_breached : self.debug("Value {} of {} for tag {} has occurred {} time(s) < threshold of {}".format( self.value, self.name(), self.get_tags(), self.get_occurrences(), self.alert_config.occurrences(), )) return "" return body def form(self, operator, static) : return "{}\n{:.2f} {}= {}\n{}".format( self.name(), self.value, operator, static, self.alert_config.url(), ) def set_tags(self, tags) : if tags : self.tags = tags elif self.result : import itertools result_tags = [ self.result['tags'][x] for x in self.alert_config.get_tags() ] chain = itertools.chain(result_tags) sorted_list = sorted(list(chain)) self.tags = ", ".join(sorted_list) if not self.tags : self.tags = "instance" def get_tags(self) : return self.tags def set_firing(self, min_value, max_value) : self.thresholds = Thresholds(self.alert_config) self.thresholds.set_breached(min_value, max_value) self.set_occurrences() self.set_new_level_breached() self.send_metrics() self.send_threshold_metrics() def get_firing(self) : return self.thresholds.get_breached() and self.occurrences_breached def get_occurrences(self) : tags = self.get_tags() return self.alert_config.get_for_tags(tags) def set_occurrences(self, force=None) : previous_occurrences = self.get_occurrences() if self.thresholds.get_breached() : new_occurrences = previous_occurrences+1 self.alert_config.set_for_tags(self.get_tags(), new_occurrences) self.occurrences_breached = self.alert_config.occurrences() <= new_occurrences if force : self.alert_config.set_for_tags(self.get_tags(), force) self.alert_config.set_for_tags(self.get_tags()+"_count", force) def send_metrics(self, *args, **kwargs) : print("send_metrics not impl") def set_new_level_breached(self) : key = self.get_tags() level = self.level() previous_level = self.alert_config.get_level(key) self.new_level_breached = level != previous_level self.alert_config.set_level(key, level) self.info("testInfo: {} {}".format( "NEW" if self.new_level_breached else "EXISTING", self.level(), )) def get_new_level_breached(self) : return self.new_level_breached def send_threshold_metrics(self) : # TODO self.send_metrics(self.alert_config.id, self.value) for level in [Thresholds.WARNING, Thresholds.CRITICAL] : for end in [Thresholds.UPPER, Thresholds.LOWER] : v, ok = self.alert_config.get_threshold(isUpper=level == Thresholds.UPPER, isWarning=end == Thresholds.WARNING) if ok : key = "{}_{}_threshold".format( "upper" if level == Thresholds.UPPER else "lower", "warning" if level == Thresholds.WARNING else "critical", ) self.send_stat(key, v, {'id':self.name()})