Skip to content

K
Kafka-Alert-Script
Commit b252508b by suhas.t
Options

Initial commit

parents
Showing with 268 additions and 0 deletions
conf/application.conf 0 → 100644
[mail_info]
sender_email = no-reply@ilens.io
receiver_email = arjun.pankajakshan@knowledgelens.com
subject = Kafka Lag
[log]
file_name= alert
path= logs/
level=DEBUG
handler= rotating_file_handler
max_bytes= 100000
back_up_count= 3
kafka-alert.sh 0 → 100644
#!/bin/bash
KAFKA_PODS=("kafka-0" "kafka-1" "kafka-2")
KAFKA_GROUPS=(rule_processing processing_agent live_data kairos_rule_exec indep_rule_exec ilens_usage_track ilens_task_track historic_data dep_rule_exec alarm_data);
THRESHOLD_LAG=4
LAG_PODS=()
LAG_GROUPS=()
LAG_VALUES=()
for i in "${KAFKA_PODS[@]}"
do
for j in "${KAFKA_GROUPS[@]}";
do
LAG=$(kubectl exec "$i" -n ilens-infra -c kafka -- kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe -group "$j" | awk '{sum += $6} END {print sum}');
if [ $LAG -ge $THRESHOLD_LAG ]
then
LAG_PODS+=("$i")
LAG_GROUPS+=("$j")
LAG_VALUES+=("$LAG")
fi
done
done
if (( ${#LAG_PODS[@]} )); then
awk -v LAG_PODS="${LAG_PODS[*]}" -v LAG_GROUPS="${LAG_GROUPS[*]}" -v LAG_VALUES="${LAG_VALUES[*]}" 'BEGIN{
print "<!DOCTYPE html>\n<html>\n<head>\n<style>\ntd, th {\nborder: 1.5px solid #1e1c1c; \npadding: 3px; \nwidth: 40mm; \nheight: 4mm; \n}"
print "\nbody {\nfont-family: Arial, Helvetica, sans-serif;\nborder-collapse: collapse;\nfont-size: 12px;\n}\ndiv {\npadding: 1mm;\npadding-bottom: 3mm;\nfont-size: 15px;\n}"
print "\ntr:nth-child(even){background-color: #f2f2f2;}\ntr:hover {background-color: #ddd;}\nth {\npadding-top: 8px;\npadding-bottom: 8px;\ntext-align: left;\nbackground-color: #8bc34a57;\ncolor: #140909;\n}\n.podname {\nwidth: 60mm;\n}\n</style>\n</head>"
print "\n<Body>\n<div>\nHi Team,<br>\n The following list is of Kafka Groups with high Consumer Lag. Please check.\n</div>\n<table>"
print "\n<thead>\n<tr>\n<th>Kafka Pod</th>\n<th>Kafka Group</th>\n<th>Lag</th>\n</tr>\n</thead>"
split(LAG_PODS,arr1," ")
split(LAG_GROUPS,arr2," ")
split(LAG_VALUES,arr3," ")
{
for(i in arr1)
{
print "<tr>"
print "<td>" arr1[i] "</td>"
print "<td>" arr2[i] "</td>"
print "<td>" arr3[i] "</td>"
print "</tr>"
}
}
}
END{
print "\n</table>"
print "\n</Body>\n</html>\n"
}' status.txt >> email.html
/usr/bin/python3 mail.py
rm -f email.html status.txt
else
echo "Lag is minimal"
rm -f email.html status.txt
fi
logger.py 0 → 100644
"""
logger utility
"""
import logging
import os
import configparser
from logging.handlers import RotatingFileHandler
from dotenv import load_dotenv
load_dotenv()
config = configparser.ConfigParser()
config.read(f"conf/application.conf")
__log_path__ = str(config["log"]["path"])
__log_level__ = str(config["log"]["level"])
__max_bytes__ = int(config["log"]["max_bytes"])
__handler_type__ = str(config["log"]["handler"])
_log_file_name__ = str(config["log"]["file_name"])
__backup_count__ = int(config["log"]["back_up_count"])
complete_log_path = os.path.join(__log_path__, _log_file_name__)
if not os.path.isdir(__log_path__):
os.makedirs(__log_path__)
def get_logger(log_file_name=complete_log_path, log_level=__log_level__, time_format="%Y-%m-%d %H:%M:%S",
handler_type=__handler_type__, max_bytes=__max_bytes__, backup_count=__backup_count__):
"""
Creates a rotating log
"""
log_file = os.path.join(log_file_name + '.log')
__logger__ = logging.getLogger(log_file_name)
__logger__.setLevel(log_level.strip().upper())
debug_formatter = '%(asctime)s - %(levelname)-6s - %(name)s - ' \
'[%(threadName)5s:%(filename)5s:%(funcName)5s():''%(lineno)s] - %(message)s'
formatter_string = '%(asctime)s - %(levelname)-6s - %(name)s - %(levelname)3s - %(message)s'
if log_level.strip().upper() == log_level:
formatter_string = debug_formatter
formatter = logging.Formatter(formatter_string, time_format)
console_handler = logging.StreamHandler()
console_handler.setFormatter(formatter)
if __logger__.hasHandlers():
__logger__.handlers.clear()
if str(handler_type).lower() == "rotating_file_handler":
# Rotating File Handler
handler = RotatingFileHandler(log_file, maxBytes=max_bytes, backupCount=backup_count)
handler.setFormatter(formatter)
if __logger__.hasHandlers():
__logger__.handlers.clear()
__logger__.addHandler(handler)
else:
# File Handler
hdlr_service = logging.FileHandler(log_file)
hdlr_service.setFormatter(formatter)
if __logger__.hasHandlers():
__logger__.handlers.clear()
__logger__.addHandler(hdlr_service)
__logger__.addHandler(console_handler)
return __logger__
logger = get_logger()
logs/alert.log 0 → 100644
This diff is collapsed. Click to expand it.
mail.py 0 → 100644
import configparser
import requests
from requests.auth import HTTPBasicAuth
from logger import logger
from dotenv import load_dotenv
load_dotenv()
from scripts.config import env_variables
config = configparser.ConfigParser()
config.read("conf/application.conf")
var = [env_variables.receiver_email]
receiver_email_list = var[0].split(",")
receiver_email = receiver_email_list
sender_email = config["mail_info"]["sender_email"]
subject = env_variables.subject
report_file = open("email.html")
html = report_file.read()
print("sending mail")
try:
email_detail = {
"base_url": "https://cloud.ilens.io/sms-util/api/v1/eim/email/send",
"user": "AllGoodNamesRGone",
"password": "comBRANSeANtamasEbICaPeC"
}
html_text = ''
# if "\n" in text:
# html_text = f"<html><p>{text}</p></html>".replace("\n", '<br>').replace("\r", '')
payload = {
"receiver_list": receiver_email,
"from_name": sender_email,
"content": '',
"subject": subject,
"gateway_id": "default",
"template": "custom_html",
"custom_html": html
}
# if html_text:
# payload["template"] = "custom_html"
resp = requests.request("POST", email_detail['base_url'], json=payload,
auth=HTTPBasicAuth(email_detail["user"], email_detail["password"]))
logger.debug(f'{resp}')
logger.debug(f'{resp.json()}')
logger.debug(f'{resp.text}')
except Exception as e:
logger.debug("Exception in send default sms:" + str(e))
print("mail sent")
new.sh 0 → 100644
#!/bin/bash
KAFKA_PODS=("kafka-0" "kafka-1" "kafka-2")
KAFKA_GROUPS=(rule_processing processing_agent live_data kairos_rule_exec indep_rule_exec ilens_usage_track ilens_task_track historic_data dep_rule_exec)
if (( ${#KAFKA_PODS[@]} )); then
awk -v LAG_PODS="${KAFKA_GROUPS[*]}" -v KAFKA_PODS="${KAFKA_PODS[*]}" 'BEGIN{
print "<!DOCTYPE html>\n<html>\n<head>\n<style>\ntd, th {\nborder: 1.5px solid #1e1c1c; \npadding: 3px; \nwidth: 40mm; \nheight: 4mm; \n}"
print "\nbody {\nfont-family: Arial, Helvetica, sans-serif;\nborder-collapse: collapse;\nfont-size: 12px;\n}\ndiv {\npadding: 1mm;\npadding-bottom: 3mm;\nfont-size: 15px;\n}"
print "\ntr:nth-child(even){background-color: #f2f2f2;}\ntr:hover {background-color: #ddd;}\nth {\npadding-top: 8px;\npadding-bottom: 8px;\ntext-align: left;\nbackground-color: #8bc34a57;\ncolor: #140909;\n}\n.podname {\nwidth: 60mm;\n}\n</style>\n</head>"
print "\n<Body>\n<div>\nHi Team,<br>\n The following list with high Kafka Consumer Lag. Please check.\n</div>\n<table>"
print "\n<thead>\n<tr>\n<th>Lag Pod</th>\n<th>Group</th>\n<th>Lag</th>\n</tr>\n</thead>"
split(LAG_PODS,arr1," ")
split(KAFKA_PODS,arr2," ")
{
for(i in arr1)
{
print "<tr>"
print "<td>" arr1[i] "</td>"
print "<td>" arr2[i] "</td>"
print "</tr>"
}
}
}
END{
print "\n</table>"
print "\n</Body>\n</html>\n"
}' status.txt >> email.html
/usr/bin/python3 mail.py
rm -f email.html status.txt
else
echo "Lag is minimal"
rm -f email.html status.txt
fi
requirements.txt 0 → 100644
configparser==5.1.0
python-dotenv==0.18.0
\ No newline at end of file
scripts/config/__init__.py 0 → 100644
"""
This file exposes configurations from config file and environments as Class Objects
"""
import shutil
if __name__ == '__main__':
from dotenv import load_dotenv
load_dotenv()
import os.path
import sys
from configparser import ConfigParser, BasicInterpolation
class EnvInterpolation(BasicInterpolation):
"""
Interpolation which expands environment variables in values.
"""
def before_get(self, parser, section, option, value, defaults):
value = super().before_get(parser, section, option, value, defaults)
if not os.path.expandvars(value).startswith('$'):
return os.path.expandvars(value)
else:
return
try:
config = ConfigParser(interpolation=EnvInterpolation())
config.read(f"conf/application.conf")
except Exception as e:
print(f"Error while loading the config: {e}")
print("Failed to Load Configuration. Exiting!!!")
sys.stdout.flush()
sys.exit()
class env_variables:
receiver_email = config.get("mail_info", "receiver_email")
subject = config.get("mail_info", "subject")
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment