743 lines
33 KiB
Bash
743 lines
33 KiB
Bash
#! /bin/bash
|
|
|
|
#############################################################
|
|
#### Prometheus Monitor install Script for Oracle Linux, ####
|
|
#### Centos/Redhat and Debian/Ubuntu Servers. ####
|
|
#### ####
|
|
#### Author: Phil Connor 08/27/2023 ####
|
|
#### Contact: pconnor@ara.com ####
|
|
#### Version 1.65.031524 ####
|
|
#### ####
|
|
#### To use this script chmod it to 755 ####
|
|
#### or simply type bash <filename.sh> ####
|
|
#############################################################
|
|
|
|
########################
|
|
### System Variables ###
|
|
########################
|
|
if [ "$(command -v lsb_release)" ]; then
|
|
OS=$(lsb_release -i | awk '{print $3}' | tr '[:upper:]' '[:lower:]')
|
|
OSVER=$(lsb_release -r | awk '{print $2}' | awk -F. '{print $1}')
|
|
else
|
|
OS=$(grep PRETTY_NAME /etc/os-release | sed 's/PRETTY_NAME=//g' | tr -d '="' | awk '{print $1}' | tr '[:upper:]' '[:lower:]')
|
|
OSVER=$(grep VERSION_ID /etc/os-release | sed 's/VERSION_ID=//g' | tr -d '="' | awk -F. '{print $1}')
|
|
fi
|
|
|
|
###########################################################
|
|
#### Detect Package Manger from OS and OSVer Variables ####
|
|
###########################################################
|
|
if [[ "${OS}" = almalinux || "${OS}" = centos || "${OS}" = red || "${OS}" = rocky ]]; then
|
|
pkgmgr="dnf -y"
|
|
elif [ "${OS}" = ubuntu ]; then
|
|
pkgmgr="apt -y"
|
|
fi
|
|
|
|
#####################################
|
|
#### Check for Service Directory ####
|
|
#####################################
|
|
if ! [ -d "/usr/lib/systemd/system/" ]; then
|
|
psdir='/etc/systemd/system/'
|
|
else
|
|
psdir='/usr/lib/systemd/system/'
|
|
fi
|
|
|
|
###########################
|
|
#### Create User/Group ####
|
|
###########################
|
|
if ! grep prometheus /etc/passwd; then
|
|
groupadd --system prometheus
|
|
useradd -s /sbin/nologin --system -g prometheus prometheus
|
|
fi
|
|
|
|
#############################
|
|
#### Check for Directory ####
|
|
#############################
|
|
if [ ! -d "/var/lib/node_exporter/" ]; then
|
|
mkdir -p /var/lib/node_exporter
|
|
chown prometheus. /var/lib/node_exporter
|
|
fi
|
|
|
|
|
|
###############################
|
|
#### Install Node Exporter ####
|
|
###############################
|
|
install_node_exporter() {
|
|
{
|
|
rm -rf /tmp/node_exporter*
|
|
cd /tmp || exit 2
|
|
curl -s https://api.github.com/repos/prometheus/node_exporter/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi -
|
|
tar -xvf node_exporter*.tar.gz
|
|
cd node_exporter*/ || exit 2
|
|
mv node_exporter /usr/local/bin
|
|
chown prometheus. /usr/local/bin/node_exporter
|
|
|
|
if [ "$OS" = red ]; then
|
|
if [ "$OSVER" = 8 ]; then
|
|
restorecon -rv /usr/local/bin/node_exporter
|
|
fi
|
|
fi
|
|
|
|
touch $psdir/node_exporter.service
|
|
{
|
|
echo '[Unit]'
|
|
echo 'Description=Prometheus Node Exporter'
|
|
echo 'Wants=network-online.target'
|
|
echo 'After=network-online.target'
|
|
echo ''
|
|
echo '[Service]'
|
|
if [ "$OS" = red ]; then
|
|
if [ "$OSVER" = 8 ]; then
|
|
echo 'User=root'
|
|
echo 'Group=root'
|
|
fi
|
|
else
|
|
echo 'User=prometheus'
|
|
echo 'Group=prometheus'
|
|
fi
|
|
echo 'Type=simple'
|
|
echo "ExecStart=/usr/local/bin/node_exporter \\"
|
|
echo " --no-collector.btrfs \\"
|
|
echo " --no-collector.hwmon \\"
|
|
echo " --no-collector.infiniband \\"
|
|
echo " --no-collector.powersupplyclass \\"
|
|
echo " --no-collector.thermal_zone \\"
|
|
echo " --no-collector.zfs \\"
|
|
echo " --collector.ethtool \\"
|
|
echo " --collector.interrupts \\"
|
|
echo " --collector.processes \\"
|
|
echo " --collector.systemd \\"
|
|
echo " --collector.tcpstat \\"
|
|
echo " --collector.textfile.directory=/var/lib/node_exporter \\"
|
|
echo ' --collector.filesystem.mount-points-exclude=^/(boot/.+|dev|proc|run|sys|run/.+|var/lib/docker/.+|var/lib/kubelet/.+|var/lib/snap/.+|volumes/user)($|/)'
|
|
echo ''
|
|
echo '[Install]'
|
|
echo 'WantedBy=multi-user.target'
|
|
} > $psdir/node_exporter.service
|
|
|
|
systemctl daemon-reload
|
|
systemctl enable --now node_exporter
|
|
#systemctl restart node_exporter
|
|
}
|
|
}
|
|
|
|
##################################
|
|
#### Install Loki Log Manager ####
|
|
##################################
|
|
install_loki () {
|
|
{
|
|
mkdir -p /etc/loki
|
|
cd /tmp || exit 2
|
|
|
|
wget https://raw.githubusercontent.com/grafana/loki/master/cmd/loki/loki-local-config.yaml
|
|
wget https://raw.githubusercontent.com/grafana/loki/main/clients/cmd/promtail/promtail-local-config.yaml
|
|
|
|
mv /tmp/loki-local-config.yaml /etc/prometheus/loki.yaml
|
|
mv /tmp/promtail-local-config.yaml /etc/prometheus/promtail.yaml
|
|
|
|
{
|
|
echo 'server:'
|
|
echo ' http_listen_port: 9080'
|
|
echo ' grpc_listen_port: 0'
|
|
echo ''
|
|
echo 'positions:'
|
|
echo ' filename: /tmp/positions.yaml'
|
|
echo ''
|
|
echo 'clients:'
|
|
echo ' - url: http://localhost:3100/loki/api/v1/push'
|
|
echo ''
|
|
echo 'scrape_configs:'
|
|
echo '- job_name: system'
|
|
echo ' static_configs:'
|
|
echo ' - targets:'
|
|
echo ' - localhost'
|
|
echo ' labels:'
|
|
echo ' job: varlogs'
|
|
echo ' __path__: /var/log/*.log # modify to point at your application log file'
|
|
echo '- job_name: access_log'
|
|
echo ' static_configs:'
|
|
echo ' - targets:'
|
|
echo ' - localhost'
|
|
echo ' - labels:'
|
|
echo ' job: apache_logs'
|
|
echo ' __path__: /var/log/http/*log'
|
|
echo ' pipeline_stages:'
|
|
echo ' - regex:'
|
|
echo ' expression: >-'
|
|
echo ' ^(?P<ip>.*?) (?P<remote_log_name>.*?)'
|
|
echo ' (?P<userid>.*?) \[(?P<date>.*?) (?P<timezone>.*?)\]'
|
|
echo ' \"(?P<request_method>.*?) (?P<path>.*?)'
|
|
echo ' (?P<request_version>HTTP/.*)?\" (?P<status>.*?)'
|
|
echo ' (?P<length>.*?) \"(?P<referrer>.*?)\"'
|
|
echo ' \"(?P<user_agent>.*?)\" (?P<session_id>.*?)'
|
|
echo ' response-time=(?P<response_time>.*)'
|
|
echo ' - labels:'
|
|
echo ' ip:'
|
|
echo ' userid:'
|
|
echo ' date:'
|
|
echo ' timezone:'
|
|
echo ' request_method:'
|
|
echo ' path:'
|
|
echo ' request_version:'
|
|
echo ' status:'
|
|
echo ' length:'
|
|
} > /etc/loki/promtail.yml
|
|
|
|
{
|
|
echo 'auth_enabled: false'
|
|
echo ''
|
|
echo 'server:'
|
|
echo ' http_listen_port: 3100'
|
|
echo ' grpc_listen_port: 9096'
|
|
echo ''
|
|
echo 'ingester:'
|
|
echo ' wal:'
|
|
echo ' enabled: true'
|
|
echo ' dir: /tmp/wal'
|
|
echo ' lifecycler:'
|
|
echo ' address: 127.0.0.1'
|
|
echo ' ring:'
|
|
echo ' kvstore:'
|
|
echo ' store: inmemory'
|
|
echo ' replication_factor: 1'
|
|
echo ' final_sleep: 0s'
|
|
echo ' chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed'
|
|
echo ' max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h'
|
|
echo ' chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first'
|
|
echo ' chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)'
|
|
echo ' max_transfer_retries: 0 # Chunk transfers disabled'
|
|
echo ''
|
|
echo 'schema_config:'
|
|
echo ' configs:'
|
|
echo ' - from: 2020-10-24'
|
|
echo ' store: boltdb-shipper'
|
|
echo ' object_store: filesystem'
|
|
echo ' schema: v11'
|
|
echo ' index:'
|
|
echo ' prefix: index'_
|
|
echo ' period: 24h'
|
|
echo ''
|
|
echo 'storage_config:'
|
|
echo ' boltdb_shipper:'
|
|
echo ' active_index_directory: /tmp/loki/boltdb-shipper-active'
|
|
echo ' cache_location: /tmp/loki/boltdb-shipper-cache'
|
|
echo ' cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space'
|
|
echo ' shared_store: filesystem'
|
|
echo ' filesystem:'
|
|
echo ' directory: /tmp/loki/chunks'
|
|
echo ''
|
|
echo 'compactor:'
|
|
echo ' working_directory: /tmp/loki/boltdb-shipper-compactor'
|
|
echo ' shared_store: filesystem'
|
|
echo ''
|
|
echo 'limits_config:'
|
|
echo ' reject_old_samples: true'
|
|
echo ' reject_old_samples_max_age: 168h'
|
|
echo ''
|
|
echo 'chunk_store_config:'
|
|
echo ' max_look_back_period: 0s'
|
|
echo ''
|
|
echo 'table_manager:'
|
|
echo ' retention_deletes_enabled: false'
|
|
echo ' retention_period: 0s'
|
|
echo ''
|
|
echo 'ruler:'
|
|
echo ' storage:'
|
|
echo ' type: local'
|
|
echo ' local:'
|
|
echo ' directory: /tmp/loki/rules'
|
|
echo ' rule_path: /tmp/loki/rules-temp'
|
|
echo ' alertmanager_url: http://localhost:9093'
|
|
echo ' ring:'
|
|
echo ' kvstore:'
|
|
echo ' store: inmemory'
|
|
echo ' enable_api: true'
|
|
} > /etc/loki/loki.yml
|
|
|
|
wget https://github.com/grafana/loki/releases/download/v2.2.1/loki-linux-amd64.zip
|
|
unzip loki-linux-amd64.zip
|
|
mv loki-linux-amd64 /usr/local/bin/loki
|
|
wget https://github.com/grafana/loki/releases/download/v2.2.1/promtail-linux-amd64.zip
|
|
unzip promtail-linux-amd64.zip
|
|
mv promtail-linux-amd64 /usr/local/bin/promtail
|
|
|
|
{
|
|
echo '[Unit]'
|
|
echo 'Description=Promtail service'
|
|
echo 'After=network.target'
|
|
echo ''
|
|
echo '[Service]'
|
|
echo 'Type=simple'
|
|
echo 'User=root'
|
|
echo 'Group=root'
|
|
echo "ExecStart=/usr/local/bin/promtail \\"
|
|
echo ' -config.file /etc/prometheus/promtail.yml'
|
|
echo 'Restart=always'
|
|
echo ''
|
|
echo '[Install]'
|
|
echo 'WantedBy=multi-user.target'
|
|
} > $psdir/promtail.service
|
|
|
|
systemctl start promtail.service
|
|
systemctl enable promtail.service
|
|
|
|
{
|
|
echo '[Unit]'
|
|
echo 'Description=Loki service'
|
|
echo 'After=network.target'
|
|
echo ''
|
|
echo '[Service]'
|
|
echo 'Type=simple'
|
|
echo 'User=prometheus'
|
|
echo 'Group=prometheus'
|
|
echo "ExecStart=/usr/local/bin/loki \\"
|
|
echo ' -config.file /etc/prometheus/loki.yml'
|
|
echo 'Restart=always'
|
|
echo ''
|
|
echo '[Install]'
|
|
echo 'WantedBy=multi-user.target'
|
|
} > $psdir/loki.service
|
|
|
|
systemctl start loki.service
|
|
systemctl enable loki.service
|
|
}
|
|
}
|
|
|
|
#############################################
|
|
#### Install Audit Directory Size Detect ####
|
|
#############################################
|
|
install_directory_size() {
|
|
{
|
|
############################
|
|
#### Check for Cron Job ####
|
|
############################
|
|
if ! crontab -l | grep -q 'directory_size.sh'; then
|
|
echo -e "$(crontab -u root -l)\n*/5 * * * * /usr/local/bin/directory_size.sh /var/log/audit > /var/lib/node_exporter/directory_size.prom 2>&1" | crontab -u root -
|
|
fi
|
|
|
|
#####################
|
|
### Create Script ###
|
|
#####################
|
|
{
|
|
# shellcheck disable=SC2028
|
|
echo '#! /bin/bash'
|
|
echo ''
|
|
echo '########################################################################################################'
|
|
echo '### ###'
|
|
echo '### Expose directory usage metrics, passed as an argument. ###'
|
|
echo '### ###'
|
|
echo '### Usage: add this to crontab: ###'
|
|
echo '### ###'
|
|
echo '### */5 * * * * directory_size.sh <directory_name> > /var/lib/node_exporter/directory_size.prom 2>&1 ###'
|
|
echo '### ###'
|
|
echo '### Phil Connor pconnor@ara.com ###'
|
|
echo '### Version 1.0.3-111623 ###'
|
|
echo '### ###'
|
|
echo '########################################################################################################'
|
|
echo ''
|
|
echo '###################################'
|
|
echo '#### Directory Metrics Command ####'
|
|
echo '###################################'
|
|
echo 'echo "# HELP node_directory_size_bytes Disk space used by some directories"'
|
|
echo 'echo "# TYPE node_directory_size_bytes gauge"'
|
|
# shellcheck disable=SC2028
|
|
echo 'du --block-size=1 --summarize "$@" | sed -ne '\''s/\\/\\\\/;s/"/\\"/g;s/^\([0-9]\+\)\t\(.*\)$/node_directory_size_bytes{directory="\2"} \1/p'\'' '
|
|
} > /usr/local/bin/directory_size.sh
|
|
|
|
chmod 755 /usr/local/bin/directory_size.sh
|
|
}
|
|
}
|
|
|
|
#####################################################
|
|
#### Install Yum/Apt Update/Reboot Needed Detect ####
|
|
#####################################################
|
|
install_apt_yum_detect() {
|
|
{
|
|
############################
|
|
#### Check for Cron Job ####
|
|
############################
|
|
if [ "$OS" = ubuntu ]; then
|
|
pkg_file=apt_detect
|
|
else
|
|
pkg_file=yum_detect
|
|
fi
|
|
|
|
if ! crontab -l | grep -q "$pkg_file.sh"; then
|
|
echo -e "$(crontab -u root -l)\n*/5 * * * * /usr/local/bin/$pkg_file.sh > /var/lib/node_exporter/$pkg_file.prom 2>&1" | crontab -u root -
|
|
fi
|
|
|
|
#####################
|
|
### Create Script ###
|
|
#####################
|
|
if [ "$OS" = ubuntu ]; then
|
|
touch /usr/local/bin/apt_detect.sh
|
|
|
|
{
|
|
echo '#!/bin/bash'
|
|
echo ''
|
|
echo '#####################################################'
|
|
echo '### ###'
|
|
echo '### Description: Expose metrics from apt updates. ###'
|
|
echo '### ###'
|
|
echo '### Phil Connor pconnor@ara.com ###'
|
|
echo '### Version 1.2.2.031524 ###'
|
|
echo '### ###'
|
|
echo '#####################################################'
|
|
# shellcheck disable=SC1003,SC2016
|
|
echo 'upgrades="$(/usr/bin/apt-get --just-print upgrade \'
|
|
# shellcheck disable=SC1003
|
|
echo ' | /usr/bin/awk -F'\''[()]'\'' \'
|
|
# shellcheck disable=SC2016
|
|
echo ' '\''/^Inst/ { sub("^[^ ]+ ", "", $2); gsub(" ","",$2);'
|
|
# shellcheck disable=SC1003,SC2016,SC2028
|
|
echo ' sub("\\[", " ", $2); sub("\\]", "", $2); print $2 }'\'' \'
|
|
# shellcheck disable=SC1003
|
|
echo ' | /usr/bin/sort \'
|
|
# shellcheck disable=SC1003
|
|
echo ' | /usr/bin/uniq -c \'
|
|
# shellcheck disable=SC2016,SC2028
|
|
echo ' | awk '\''{ gsub(/\\\\/, "\\\\", $2); gsub(/\\"/, "\\\"", $2);'
|
|
# shellcheck disable=SC2016
|
|
echo ' gsub(/\[/, "", $3); gsub(/\]/, "", $3);'
|
|
# shellcheck disable=SC2016
|
|
echo ' print "apt_upgrades_pending{origin=\"" $2 "\",arch=\"" $3 "\"} " $1}'\'' '
|
|
echo ')"'
|
|
echo ''
|
|
# shellcheck disable=SC2016
|
|
echo 'upgradelist=$(/usr/bin/apt-get --just-print upgrade | grep Inst | awk '\''{gsub(/\(|\)/,"",$4)}1 {gsub(/:/,".",$4)}1 \ '
|
|
# shellcheck disable=SC2016
|
|
echo '{gsub(/\[|\]/,"",$3)}1 {gsub(/:/," ",$5)}1 {print "apt_upgradelist{pkgname=\"" $2 "\",uvers=\"" $4 "\", overs=\"" $3 "\", repo=\"" $5 "\"} " 1}'\'')'
|
|
echo ''
|
|
echo 'echo '\''# HELP apt_upgrades_pending Apt package pending updates by origin.'\'' '
|
|
echo 'echo '\''# TYPE apt_upgrades_pending gauge'\'' '
|
|
# shellcheck disable=SC2016
|
|
echo 'if [[ -n "${upgrades}" ]] ; then'
|
|
# shellcheck disable=SC2016
|
|
echo ' echo "${upgrades}"'
|
|
echo 'else'
|
|
echo ' echo '\''apt_upgrades_pending{origin="",arch=""} 0'\'' '
|
|
echo 'fi'
|
|
echo ''
|
|
echo 'echo '\''# HELP apt_upgradelist List of packages for upgrade'\'' '
|
|
echo 'echo '\''# TYPE apt_upgradelist gauge'\'' '
|
|
# shellcheck disable=SC2016
|
|
echo 'if [[ -n "${upgradelist}" ]]; then'
|
|
# shellcheck disable=SC2016
|
|
echo ' echo "${upgradelist}"'
|
|
echo 'else'
|
|
echo ' echo '\''apt_upgradelist{origin="",arch=""} 0'\'' '
|
|
echo 'fi'
|
|
echo 'echo '\''# HELP node_reboot_required Node reboot is required for software updates.'\'' '
|
|
echo '# TYPE node_reboot_required gauge'
|
|
echo 'if [[ -f '/run/reboot-required' ]] ; then'
|
|
echo ' echo '\''node_reboot_required 1'\'' '
|
|
echo 'else'
|
|
echo ' echo '\''node_reboot_required 0'\'' '
|
|
echo 'fi'
|
|
} > /usr/local/bin/apt_detect.sh
|
|
|
|
chmod 755 /usr/local/bin/apt_detect.sh
|
|
else
|
|
touch /usr/local/bin/yum_detect.sh
|
|
|
|
{
|
|
echo '#! /bin/bash'
|
|
echo ''
|
|
echo '#####################################################'
|
|
echo '### ###'
|
|
echo '### Description: Expose metrics from yum updates. ###'
|
|
echo '### ###'
|
|
echo '### Phil Connor pconnor@ara.com ###'
|
|
echo '### Version 1.1.5.031524 ###'
|
|
echo '### ###'
|
|
echo '#####################################################'
|
|
echo 'set -u -o pipefail'
|
|
echo ''
|
|
echo '# shellcheck disable=SC2016'
|
|
echo 'filter_awk_script='\'' '
|
|
echo 'BEGIN { mute=1 }'
|
|
echo '/Obsoleting Packages/ {'
|
|
echo ' mute=0'
|
|
echo '}'
|
|
echo 'mute && /^[[:print:]]+\.[[:print:]]+/ {'
|
|
# shellcheck disable=SC2016
|
|
echo ' print $3'
|
|
echo '}'
|
|
echo ''\'' '
|
|
echo ''
|
|
echo 'check_upgrades() {'
|
|
echo ' {'
|
|
echo ' /usr/bin/yum -q check-update |'
|
|
echo ' /usr/bin/xargs -n3 |'
|
|
# shellcheck disable=SC2016
|
|
echo ' awk "${filter_awk_script}" |'
|
|
echo ' sort |'
|
|
echo ' uniq -c |'
|
|
# shellcheck disable=SC2016
|
|
echo ' awk '\''{print "yum_upgrades_pending{origin=\""$2"\"} "$1}'\'' '
|
|
echo ' }'
|
|
echo '}'
|
|
echo ''
|
|
# shellcheck disable=SC2016
|
|
echo 'upgrades=$(check_upgrades)'
|
|
echo ''
|
|
echo 'echo '\''# HELP yum_upgrades_pending Yum package pending updates by origin.'\'' '
|
|
echo 'echo '\''# TYPE yum_upgrades_pending gauge'\'' '
|
|
# shellcheck disable=SC2016
|
|
echo 'if [[ -n "${upgrades}" ]]; then'
|
|
# shellcheck disable=SC2016
|
|
echo ' echo "${upgrades}"'
|
|
echo 'else'
|
|
echo ' echo '\''yum_upgrades_pending{origin=""} 0'\'' '
|
|
echo 'fi'
|
|
echo ''
|
|
# shellcheck disable=SC2016
|
|
echo 'upgradelist=$(yum check-update | awk '\''NR>4 {print "yum_upgradelist{pkgname=\"" $1 "\", uvers=\"" $2 "\", repo=\"" $3 "\"}", 1}'\'')'
|
|
echo ''
|
|
echo 'echo '\''# HELP yum_upgradelist List of packages for upgrade'\'' '
|
|
echo 'echo '\''# TYPE yum_upgradelist gauge'\'' '
|
|
# shellcheck disable=SC2016
|
|
echo 'if [[ -n "${upgradelist}" ]]; then'
|
|
# shellcheck disable=SC2016
|
|
echo ' echo "${upgradelist}"'
|
|
echo 'else'
|
|
echo ' echo '\''yum_upgradelist{pkgname=""} 0'\'' '
|
|
echo 'fi'
|
|
echo ''
|
|
echo '### If yum-utils/dnf-utils is not installed this will skip reboot required metric'
|
|
echo 'if [[ -x /bin/needs-restarting ]]; then'
|
|
echo ' echo '\''# HELP node_reboot_required Node reboot is required for software updates.'\'' '
|
|
echo ' echo '\''# TYPE node_reboot_required gauge'\'' '
|
|
echo ' if /bin/needs-restarting -r > /dev/null 2>&1; then'
|
|
echo ' echo '\''node_reboot_required 0'\'' '
|
|
echo ' else'
|
|
echo ' echo '\''node_reboot_required 1'\'' '
|
|
echo ' fi'
|
|
echo 'fi'
|
|
} > /usr/local/bin/yum_detect.sh
|
|
|
|
chmod 755 /usr/local/bin/yum_detect.sh
|
|
fi
|
|
}
|
|
}
|
|
|
|
install_salt_status() {
|
|
{
|
|
if [ "$(command -v /bin/ss)" ]; then
|
|
ssdir=/bin
|
|
elif [ "$(command -v /usr/bin/ss)" ]; then
|
|
ssdir=/usr/bin
|
|
else
|
|
ssdir=/usr/sbin
|
|
fi
|
|
|
|
############################
|
|
#### Check for Cron Job ####
|
|
############################
|
|
if ! crontab -l | grep -q "salt_status.sh"; then
|
|
echo -e "$(crontab -u root -l)\n*/5 * * * * /usr/local/bin/salt_status.sh > /var/lib/node_exporter/salt_status.prom 2>&1" | crontab -u root -
|
|
fi
|
|
|
|
#####################
|
|
### Create Script ###
|
|
#####################
|
|
touch /usr/local/bin/salt_status.sh
|
|
|
|
{
|
|
echo '#! /bin/bash'
|
|
echo '#####################################################'
|
|
echo '### ###'
|
|
echo '### Description: Expose metrics from salt-minion. ###'
|
|
echo '### ###'
|
|
echo '### Phil Connor pconnor@ara.com ###'
|
|
echo '### Version 1.3.3.122023 ###'
|
|
echo '### ###'
|
|
echo '#####################################################'
|
|
echo ''
|
|
echo '## Local Command Variables'
|
|
# shellcheck disable=SC2016
|
|
echo 'if [ ! "$(command -v salt-call)" ]; then'
|
|
echo ' saltck=none'
|
|
echo 'else'
|
|
# shellcheck disable=SC2016,SC2028
|
|
echo ' saltck=$(salt-call test.ping | grep '\''\bTrue\b'\'')'
|
|
echo 'fi'
|
|
echo ''
|
|
# shellcheck disable=SC2016,SC2028
|
|
echo "status=\$($ssdir/ss -nt | grep '\b4505\b')"
|
|
echo ''
|
|
echo '## Check If minion is connected to port 4505'
|
|
echo 'echo '\''# HELP minion_connection_status Shows if Salt-Minion is connected to Salt-Master.'\'' '
|
|
echo 'echo '\''# TYPE minion_connection_status gauge'\'' '
|
|
# shellcheck disable=SC2016
|
|
echo 'if [[ -n "${status}" ]]; then'
|
|
echo ' echo '\''minion_connection_status{status=""} 1'\'' '
|
|
echo 'else'
|
|
echo ' echo '\''minion_connection_status{status=""} 0'\'' '
|
|
echo 'fi'
|
|
echo ''
|
|
echo '## Check to see in minion can ping master'
|
|
echo 'echo '\''# HELP minion_ping_status Shows if Salt-Minion is able to ping Salt-Master.'\'' '
|
|
echo 'echo '\''# TYPE minion_ping_status gauge'\'' '
|
|
# shellcheck disable=SC2016
|
|
echo 'if [ $saltck = "error" ]; then'
|
|
echo ' echo '\''minion_ping_status{status=""} 3'\'' '
|
|
# shellcheck disable=SC2016
|
|
echo 'elif [ $saltck = none ]; then'
|
|
echo ' echo '\''minion_ping_status{status=""} 2'\'' '
|
|
# shellcheck disable=SC2016
|
|
echo 'elif [[ -n "${saltck}" ]]; then'
|
|
echo ' echo '\''minion_ping_status{status=""} 1'\'' '
|
|
echo 'else'
|
|
echo ' echo '\''minion_ping_status{status=""} 0'\'' '
|
|
echo 'fi'
|
|
} > /usr/local/bin/salt_status.sh
|
|
|
|
chmod 755 /usr/local/bin/salt_status.sh
|
|
}
|
|
}
|
|
|
|
install_process_list() {
|
|
{
|
|
############################
|
|
#### Check for Cron Job ####
|
|
############################
|
|
if ! crontab -l | grep -q "process_list.sh"; then
|
|
echo -e "$(crontab -u root -l)\n*/3 * * * * /usr/local/bin/process_list.sh > /var/lib/node_exporter/process_list.prom 2>&1" | crontab -u root -
|
|
fi
|
|
|
|
##################################
|
|
### Check if lsof is installed ###
|
|
##################################
|
|
if [ ! "$(command -v lsof)" ]; then
|
|
$pkgmgr install lsof
|
|
fi
|
|
|
|
##########################
|
|
### Set lsof Dirictory ###
|
|
##########################
|
|
if [ "$(command -v /bin/lsof)" ]; then
|
|
lfdir=/bin
|
|
elif [ "$(command -v /usr/bin/lsof)" ]; then
|
|
lfdir=/usr/bin
|
|
else
|
|
lfdir=/usr/sbin
|
|
fi
|
|
|
|
#####################
|
|
### Create Script ###
|
|
#####################
|
|
touch /usr/local/bin/process_list.sh
|
|
|
|
{
|
|
echo '#! /bin/bash'
|
|
echo ''
|
|
echo '########################################################'
|
|
echo '### ###'
|
|
echo '### Description: Expose metrics from cpu by process. ###'
|
|
echo '### ###'
|
|
echo '### Phil Connor pconnor@ara.com ###'
|
|
echo '### Version 2.8.1.031424 ###'
|
|
echo '### ###'
|
|
echo '########################################################'
|
|
echo ''
|
|
echo '#############################'
|
|
echo '### Process List Function ###'
|
|
echo '#############################'
|
|
echo 'processes_list() {'
|
|
echo ' {'
|
|
# shellcheck disable=SC2016
|
|
echo ' PList=$(ps aux)'
|
|
echo ''
|
|
echo ' while read -r PList'
|
|
echo ' do'
|
|
# shellcheck disable=SC2016
|
|
echo ' pl=$(/usr/bin/awk '\''{print "node_cpu_usage{process=\""$11"\", pid=\""$2"\", owner=\""$1"\"}", $3}'\'')'
|
|
# shellcheck disable=SC2016
|
|
echo ' done <<< "$PList"'
|
|
echo ''
|
|
echo ' echo '\''# HELP node_cpu_usage Usage of CPU by process.'\'' '
|
|
echo ' echo '\''# TYPE node_cpu_usage gauge'\'' '
|
|
# shellcheck disable=SC2016
|
|
echo ' echo "$pl"'
|
|
echo ' }'
|
|
echo '}'
|
|
echo ''
|
|
echo '##################################'
|
|
echo '### File Handler List Function ###'
|
|
echo '##################################'
|
|
echo 'filehandlers_list() {'
|
|
echo ' {'
|
|
# shellcheck disable=SC2016,SC2028
|
|
echo " Fhlist=\$($lfdir/lsof | /usr/bin/awk '{gsub(/\\\\/,\"\",\$1)}1 {gsub(/\\\\|\:/,\"\",\$4)}1 {print \$1 \" \" \$2 \" \" \$4}' | sort | uniq -c | sort -rn | head -20)"
|
|
echo ''
|
|
echo ' while read -r Fhlist'
|
|
echo ' do'
|
|
# shellcheck disable=SC2016
|
|
echo ' fhl=$(/usr/bin/awk '\''{print "node_file_handlers{pid=\""$1"\", program=\""$2"\"}", $3}'\'')'
|
|
# shellcheck disable=SC2016
|
|
echo ' done <<< "$Fhlist"'
|
|
echo ' echo '\''# HELP node_file_handler Usage of File Handlers.'\'' '
|
|
echo ' echo '\''# TYPE node_file_handler gauge'\'' '
|
|
# shellcheck disable=SC2016
|
|
echo ' echo "$fhl"'
|
|
echo ''
|
|
# shellcheck disable=SC2016
|
|
echo ' fhmax=$(cat /proc/sys/fs/file-nr | awk '\''{print $3}'\'')'
|
|
# shellcheck disable=SC2016
|
|
echo ' fhfree=$(cat /proc/sys/fs/file-nr | awk '\''{print $2}'\'')'
|
|
# shellcheck disable=SC2016
|
|
echo ' fhopen=$(cat /proc/sys/fs/file-nr | awk '\''{print $1}'\'')'
|
|
echo ' echo '\''# HELP node_max_files Max File Limit Handlers.'\'' '
|
|
echo ' echo '\''# TYPE node_max_files gauge'\'' '
|
|
# shellcheck disable=SC2016
|
|
echo ' echo "node_total_max_files_handles $fhmax"'
|
|
# shellcheck disable=SC2016
|
|
echo ' echo "node_total_free_file_handles $fhfree"'
|
|
# shellcheck disable=SC2016
|
|
echo ' echo "node_total_open_file_handles $fhopen"'
|
|
echo ''
|
|
echo ' }'
|
|
echo '}'
|
|
echo ''
|
|
echo '################################'
|
|
echo '### File Handler Connections ###'
|
|
echo '################################'
|
|
echo 'filehandler_connections() {'
|
|
echo ' {'
|
|
# shellcheck disable=SC2016,SC2028
|
|
echo " fhconn=\$($lfdir/lsof -i | /usr/bin/awk '{gsub(/\\\\/,\"_\",\$1)}1 {gsub(/\\*|\\:/\"_\",\$2)}1 {gsub(/\\*|\\:|\\\\/,\"\",\$9)}1')"
|
|
echo ''
|
|
echo ' while read -r fhconn'
|
|
echo ' do'
|
|
# shellcheck disable=SC2016
|
|
echo ' conn=$(/usr/bin/awk '\''NR>1 {print "node_file_handle_connection{command=\""$1"\", connection=\""$9"\", user=\""$3"\", protocol=\""$5"\", type=\""$8"\"}", $2}'\'')'
|
|
# shellcheck disable=SC2016
|
|
echo ' done <<< "$fhconn"'
|
|
echo ' echo '\''# HELP node_file_handle_connection Connections by process.'\'' '
|
|
echo ' echo '\''# TYPE node_file_handle_connection gauge'\'' '
|
|
# shellcheck disable=SC2016
|
|
echo ' echo "$conn"'
|
|
echo ' }'
|
|
echo '}'
|
|
echo ''
|
|
echo '######################'
|
|
echo '### Function Calls ###'
|
|
echo '######################'
|
|
echo 'processes_list'
|
|
echo 'filehandlers_list'
|
|
echo 'filehandler_connections'
|
|
} > /usr/local/bin/process_list.sh
|
|
|
|
chmod 755 /usr/local/bin/process_list.sh
|
|
|
|
}
|
|
}
|
|
########################
|
|
#### Function Calls ####
|
|
########################
|
|
install_node_exporter
|
|
#install_loki
|
|
#install_directory_size
|
|
install_apt_yum_detect
|
|
install_salt_status
|
|
install_process_list |