diff --git a/apt_detect.sh b/apt_detect.sh new file mode 100644 index 0000000..8389923 --- /dev/null +++ b/apt_detect.sh @@ -0,0 +1,30 @@ +#!/bin/bash +######################################### +# Description: Expose metrics from apt updates. +# +# Phil Connor contact@mylinux.work +# Version 1.0.1.103123 +######################################### + +upgrades="$(/usr/bin/apt-get --just-print upgrade | /usr/bin/awk -F'[()]' \ + '/^Inst/ { sub("^[^ ]+ ", "", $2); gsub(" ","",$2); sub("\\[", " ", $2); sub("\\]", "", $2); print $2 }' \ + | /usr/bin/sort | /usr/bin/uniq -c | awk '{ gsub(/\\\\/, "\\\\", $2); gsub(/\\"/, "\\\"", $2); + gsub(/\[/, "", $3); gsub(/\]/, "", $3); + print "apt_upgrades_pending{origin=\"" $2 "\",arch=\"" $3 "\"} " $1}' +)" + +echo '# HELP apt_upgrades_pending Apt package pending updates by origin.' +echo '# TYPE apt_upgrades_pending gauge' +if [[ -n "${upgrades}" ]] ; then + echo "${upgrades}" +else + echo 'apt_upgrades_pending{origin="",arch=""} 0' +fi + +echo '# HELP node_reboot_required Node reboot is required for software updates.' +echo '# TYPE node_reboot_required gauge' +if [[ -f '/run/reboot-required' ]] ; then + echo 'node_reboot_required 1' +else + echo 'node_reboot_required 0' +fi \ No newline at end of file diff --git a/directory_size.sh b/directory_size.sh new file mode 100644 index 0000000..5dea74b --- /dev/null +++ b/directory_size.sh @@ -0,0 +1,35 @@ +#! /bin/bash +########################### +# Expose directory usage metrics, passed as an argument. +# +# Usage: add this to crontab: +# +# */5 * * * * directory_size.sh > /var/lib/node_exporter/directory_size.prom +# +# sed pattern taken from https://www.robustperception.io/monitoring-directory-sizes-with-the-textfile-collector/! +# +# Phil Connor pconnor@ara.com +# Version 1.0.3-111623 +########################### + +############################# +#### Check for Directory #### +############################# +if [ ! -d "/var/lib/node_exporter/" ]; then + mkdir -p /var/lib/node_exporter + chown prometheus. /var/lib/node_exporter +fi + +############################ +#### Check for Cron Job #### +############################ +if ! crontab -l | grep -q 'directory_size.sh'; then + echo -e "$(crontab -u root -l)\n*/5 * * * * /usr/local/bin/directory_size.sh /var/log/audit > /var/lib/node_exporter/directory_size.prom 2>&1" | crontab -u root - +fi + +################################### +#### Directory Metrics Command #### +################################### +echo "# HELP node_directory_size_bytes Disk space used by some directories" +echo "# TYPE node_directory_size_bytes gauge" +du --block-size=1 --summarize "$@" | sed -ne 's/\\/\\\\/;s/"/\\"/g;s/^\([0-9]\+\)\t\(.*\)$/node_directory_size_bytes{directory="\2"} \1/p' diff --git a/install_f2b_metrics.sh b/install_f2b_metrics.sh new file mode 100644 index 0000000..bc1ac30 --- /dev/null +++ b/install_f2b_metrics.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +############################################################# +#### Fail2Ban Metrics install Script for Prometheus #### +#### Centos/Redhat and Debian/Ubuntu Servers. #### +#### #### +#### Author: Phil Connor 08/27/2023 #### +#### Contact: contact@mylinux.work #### +#### Version 1.92.011924 #### +#### #### +#### To use this script chmod it to 755 #### +#### or simply type bash #### +############################################################# + + + + +if ! crontab -l | grep -q "fail2ban_metrics.sh"; then + echo -e "$(crontab -u root -l)\n*/5 * * * * /usr/local/bin/fail2ban_metrics.sh > /var/lib/node_exporter/fail2ban_metrics.prom 2>&1" | crontab -u root - +fi + +install_fail2_metrics() { + { + { + echo 'jails=0' + echo '' + # shellcheck disable=SC2016,SC2028 + echo 'for jail in $(fail2ban-client status | grep '\''Jail list:'\'' | sed '\''s/.*Jail list:[\t ]*//;s/,//g'\'')' + echo ' do' + # shellcheck disable=SC2004,SC2016,SC2030 + echo ' jails=$(($jails + 1))' + # shellcheck disable=SC2016 + echo ' fail2ban-client status "$jail" | awk -F: '\'' ' + # shellcheck disable=SC2016 + echo ' $1 ~ /Currently failed/ {' + # shellcheck disable=SC2016,SC2154 + echo ' print "fail2ban_failed_current{jail=\"'\''"$jail"'\''\"} " $2;' + echo ' }' + # shellcheck disable=SC2016 + echo ' $1 ~ /Total failed/ {' + # shellcheck disable=SC2016 + echo ' print "fail2ban_failed_total{jail=\"'\''"$jail"'\''\"} " $2;' + echo ' }' + # shellcheck disable=SC2016 + echo ' $1 ~ /Currently banned/ {' + # shellcheck disable=SC2016 + echo ' print "fail2ban_banned_current{jail=\"'\''"$jail"'\''\"} " $2;' + echo ' }' + # shellcheck disable=SC2016 + echo ' $1 ~ /Total banned/ {' + # shellcheck disable=SC2016 + echo ' print "fail2ban_banned_total{jail=\"'\''"$jail"'\''\"} " $2;' + echo ' }'\'' ' + echo ' done | sort | awk '\'' ' + echo '' + echo 'BEGIN {failc=1; failt=1; banc=1; bant=1}' + echo '/^fail2ban_failed_current/ { if (failc) {' + echo ' print "# HELP fail2ban_failed_current Current number of failures.";' + echo ' print "# TYPE fail2ban_failed_current gauge";' + echo ' failc=0;' + echo ' } }' + echo '/^fail2ban_failed_total/ { if (failt) {' + echo ' print "# HELP fail2ban_failed_total Total number of failures.";' + echo ' print "# TYPE fail2ban_failed_total counter";' + echo ' failt=0;' + echo ' } }' + echo '/^fail2ban_banned_current/ { if (banc) {' + echo ' print "# HELP fail2ban_banned_current Current number banned.";' + echo ' print "# TYPE fail2ban_banned_current gauge";' + echo ' banc=0;' + echo ' } }' + echo '/^fail2ban_banned_total/ { if (bant) {' + echo ' print "# HELP fail2ban_banned_total Total number banned.";' + echo ' print "# TYPE fail2ban_banned_total counter";' + echo ' bant=0;' + echo ' } }' + # shellcheck disable=SC2016 + echo '{ print $0 }'\'' ' + echo '' + echo 'cat < /usr/local/bin/fail2ban_metrics.sh + + chmod 755 /usr/local/bin/fail2ban_metrics.sh + } +} + +install_fail2_metrics \ No newline at end of file diff --git a/monitor_install.sh b/monitor_install.sh new file mode 100644 index 0000000..d029ea2 --- /dev/null +++ b/monitor_install.sh @@ -0,0 +1,743 @@ +#! /bin/bash + +############################################################# +#### Prometheus Monitor install Script for Oracle Linux, #### +#### Centos/Redhat and Debian/Ubuntu Servers. #### +#### #### +#### Author: Phil Connor 08/27/2023 #### +#### Contact: pconnor@ara.com #### +#### Version 1.65.031524 #### +#### #### +#### To use this script chmod it to 755 #### +#### or simply type bash #### +############################################################# + +######################## +### System Variables ### +######################## +if [ "$(command -v lsb_release)" ]; then + OS=$(lsb_release -i | awk '{print $3}' | tr '[:upper:]' '[:lower:]') + OSVER=$(lsb_release -r | awk '{print $2}' | awk -F. '{print $1}') +else + OS=$(grep PRETTY_NAME /etc/os-release | sed 's/PRETTY_NAME=//g' | tr -d '="' | awk '{print $1}' | tr '[:upper:]' '[:lower:]') + OSVER=$(grep VERSION_ID /etc/os-release | sed 's/VERSION_ID=//g' | tr -d '="' | awk -F. '{print $1}') +fi + +########################################################### +#### Detect Package Manger from OS and OSVer Variables #### +########################################################### +if [[ "${OS}" = almalinux || "${OS}" = centos || "${OS}" = red || "${OS}" = rocky ]]; then + pkgmgr="dnf -y" +elif [ "${OS}" = ubuntu ]; then + pkgmgr="apt -y" +fi + +##################################### +#### Check for Service Directory #### +##################################### +if ! [ -d "/usr/lib/systemd/system/" ]; then + psdir='/etc/systemd/system/' +else + psdir='/usr/lib/systemd/system/' +fi + +########################### +#### Create User/Group #### +########################### +if ! grep prometheus /etc/passwd; then + groupadd --system prometheus + useradd -s /sbin/nologin --system -g prometheus prometheus +fi + +############################# +#### Check for Directory #### +############################# +if [ ! -d "/var/lib/node_exporter/" ]; then + mkdir -p /var/lib/node_exporter + chown prometheus. /var/lib/node_exporter +fi + + +############################### +#### Install Node Exporter #### +############################### +install_node_exporter() { + { + rm -rf /tmp/node_exporter* + cd /tmp || exit 2 + curl -s https://api.github.com/repos/prometheus/node_exporter/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi - + tar -xvf node_exporter*.tar.gz + cd node_exporter*/ || exit 2 + mv node_exporter /usr/local/bin + chown prometheus. /usr/local/bin/node_exporter + + if [ "$OS" = red ]; then + if [ "$OSVER" = 8 ]; then + restorecon -rv /usr/local/bin/node_exporter + fi + fi + + touch $psdir/node_exporter.service + { + echo '[Unit]' + echo 'Description=Prometheus Node Exporter' + echo 'Wants=network-online.target' + echo 'After=network-online.target' + echo '' + echo '[Service]' + if [ "$OS" = red ]; then + if [ "$OSVER" = 8 ]; then + echo 'User=root' + echo 'Group=root' + fi + else + echo 'User=prometheus' + echo 'Group=prometheus' + fi + echo 'Type=simple' + echo "ExecStart=/usr/local/bin/node_exporter \\" + echo " --no-collector.btrfs \\" + echo " --no-collector.hwmon \\" + echo " --no-collector.infiniband \\" + echo " --no-collector.powersupplyclass \\" + echo " --no-collector.thermal_zone \\" + echo " --no-collector.zfs \\" + echo " --collector.ethtool \\" + echo " --collector.interrupts \\" + echo " --collector.processes \\" + echo " --collector.systemd \\" + echo " --collector.tcpstat \\" + echo " --collector.textfile.directory=/var/lib/node_exporter \\" + echo ' --collector.filesystem.mount-points-exclude=^/(boot/.+|dev|proc|run|sys|run/.+|var/lib/docker/.+|var/lib/kubelet/.+|var/lib/snap/.+|volumes/user)($|/)' + echo '' + echo '[Install]' + echo 'WantedBy=multi-user.target' + } > $psdir/node_exporter.service + + systemctl daemon-reload + systemctl enable --now node_exporter + #systemctl restart node_exporter + } +} + +################################## +#### Install Loki Log Manager #### +################################## +install_loki () { + { + mkdir -p /etc/loki + cd /tmp || exit 2 + + wget https://raw.githubusercontent.com/grafana/loki/master/cmd/loki/loki-local-config.yaml + wget https://raw.githubusercontent.com/grafana/loki/main/clients/cmd/promtail/promtail-local-config.yaml + + mv /tmp/loki-local-config.yaml /etc/prometheus/loki.yaml + mv /tmp/promtail-local-config.yaml /etc/prometheus/promtail.yaml + + { + echo 'server:' + echo ' http_listen_port: 9080' + echo ' grpc_listen_port: 0' + echo '' + echo 'positions:' + echo ' filename: /tmp/positions.yaml' + echo '' + echo 'clients:' + echo ' - url: http://localhost:3100/loki/api/v1/push' + echo '' + echo 'scrape_configs:' + echo '- job_name: system' + echo ' static_configs:' + echo ' - targets:' + echo ' - localhost' + echo ' labels:' + echo ' job: varlogs' + echo ' __path__: /var/log/*.log # modify to point at your application log file' + echo '- job_name: access_log' + echo ' static_configs:' + echo ' - targets:' + echo ' - localhost' + echo ' - labels:' + echo ' job: apache_logs' + echo ' __path__: /var/log/http/*log' + echo ' pipeline_stages:' + echo ' - regex:' + echo ' expression: >-' + echo ' ^(?P.*?) (?P.*?)' + echo ' (?P.*?) \[(?P.*?) (?P.*?)\]' + echo ' \"(?P.*?) (?P.*?)' + echo ' (?PHTTP/.*)?\" (?P.*?)' + echo ' (?P.*?) \"(?P.*?)\"' + echo ' \"(?P.*?)\" (?P.*?)' + echo ' response-time=(?P.*)' + echo ' - labels:' + echo ' ip:' + echo ' userid:' + echo ' date:' + echo ' timezone:' + echo ' request_method:' + echo ' path:' + echo ' request_version:' + echo ' status:' + echo ' length:' + } > /etc/loki/promtail.yml + + { + echo 'auth_enabled: false' + echo '' + echo 'server:' + echo ' http_listen_port: 3100' + echo ' grpc_listen_port: 9096' + echo '' + echo 'ingester:' + echo ' wal:' + echo ' enabled: true' + echo ' dir: /tmp/wal' + echo ' lifecycler:' + echo ' address: 127.0.0.1' + echo ' ring:' + echo ' kvstore:' + echo ' store: inmemory' + echo ' replication_factor: 1' + echo ' final_sleep: 0s' + echo ' chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed' + echo ' max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h' + echo ' chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first' + echo ' chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)' + echo ' max_transfer_retries: 0 # Chunk transfers disabled' + echo '' + echo 'schema_config:' + echo ' configs:' + echo ' - from: 2020-10-24' + echo ' store: boltdb-shipper' + echo ' object_store: filesystem' + echo ' schema: v11' + echo ' index:' + echo ' prefix: index'_ + echo ' period: 24h' + echo '' + echo 'storage_config:' + echo ' boltdb_shipper:' + echo ' active_index_directory: /tmp/loki/boltdb-shipper-active' + echo ' cache_location: /tmp/loki/boltdb-shipper-cache' + echo ' cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space' + echo ' shared_store: filesystem' + echo ' filesystem:' + echo ' directory: /tmp/loki/chunks' + echo '' + echo 'compactor:' + echo ' working_directory: /tmp/loki/boltdb-shipper-compactor' + echo ' shared_store: filesystem' + echo '' + echo 'limits_config:' + echo ' reject_old_samples: true' + echo ' reject_old_samples_max_age: 168h' + echo '' + echo 'chunk_store_config:' + echo ' max_look_back_period: 0s' + echo '' + echo 'table_manager:' + echo ' retention_deletes_enabled: false' + echo ' retention_period: 0s' + echo '' + echo 'ruler:' + echo ' storage:' + echo ' type: local' + echo ' local:' + echo ' directory: /tmp/loki/rules' + echo ' rule_path: /tmp/loki/rules-temp' + echo ' alertmanager_url: http://localhost:9093' + echo ' ring:' + echo ' kvstore:' + echo ' store: inmemory' + echo ' enable_api: true' + } > /etc/loki/loki.yml + + wget https://github.com/grafana/loki/releases/download/v2.2.1/loki-linux-amd64.zip + unzip loki-linux-amd64.zip + mv loki-linux-amd64 /usr/local/bin/loki + wget https://github.com/grafana/loki/releases/download/v2.2.1/promtail-linux-amd64.zip + unzip promtail-linux-amd64.zip + mv promtail-linux-amd64 /usr/local/bin/promtail + + { + echo '[Unit]' + echo 'Description=Promtail service' + echo 'After=network.target' + echo '' + echo '[Service]' + echo 'Type=simple' + echo 'User=root' + echo 'Group=root' + echo "ExecStart=/usr/local/bin/promtail \\" + echo ' -config.file /etc/prometheus/promtail.yml' + echo 'Restart=always' + echo '' + echo '[Install]' + echo 'WantedBy=multi-user.target' + } > $psdir/promtail.service + + systemctl start promtail.service + systemctl enable promtail.service + + { + echo '[Unit]' + echo 'Description=Loki service' + echo 'After=network.target' + echo '' + echo '[Service]' + echo 'Type=simple' + echo 'User=prometheus' + echo 'Group=prometheus' + echo "ExecStart=/usr/local/bin/loki \\" + echo ' -config.file /etc/prometheus/loki.yml' + echo 'Restart=always' + echo '' + echo '[Install]' + echo 'WantedBy=multi-user.target' + } > $psdir/loki.service + + systemctl start loki.service + systemctl enable loki.service + } +} + +############################################# +#### Install Audit Directory Size Detect #### +############################################# +install_directory_size() { + { + ############################ + #### Check for Cron Job #### + ############################ + if ! crontab -l | grep -q 'directory_size.sh'; then + echo -e "$(crontab -u root -l)\n*/5 * * * * /usr/local/bin/directory_size.sh /var/log/audit > /var/lib/node_exporter/directory_size.prom 2>&1" | crontab -u root - + fi + + ##################### + ### Create Script ### + ##################### + { + # shellcheck disable=SC2028 + echo '#! /bin/bash' + echo '' + echo '########################################################################################################' + echo '### ###' + echo '### Expose directory usage metrics, passed as an argument. ###' + echo '### ###' + echo '### Usage: add this to crontab: ###' + echo '### ###' + echo '### */5 * * * * directory_size.sh > /var/lib/node_exporter/directory_size.prom 2>&1 ###' + echo '### ###' + echo '### Phil Connor pconnor@ara.com ###' + echo '### Version 1.0.3-111623 ###' + echo '### ###' + echo '########################################################################################################' + echo '' + echo '###################################' + echo '#### Directory Metrics Command ####' + echo '###################################' + echo 'echo "# HELP node_directory_size_bytes Disk space used by some directories"' + echo 'echo "# TYPE node_directory_size_bytes gauge"' + # shellcheck disable=SC2028 + echo 'du --block-size=1 --summarize "$@" | sed -ne '\''s/\\/\\\\/;s/"/\\"/g;s/^\([0-9]\+\)\t\(.*\)$/node_directory_size_bytes{directory="\2"} \1/p'\'' ' + } > /usr/local/bin/directory_size.sh + + chmod 755 /usr/local/bin/directory_size.sh + } +} + +##################################################### +#### Install Yum/Apt Update/Reboot Needed Detect #### +##################################################### +install_apt_yum_detect() { + { + ############################ + #### Check for Cron Job #### + ############################ + if [ "$OS" = ubuntu ]; then + pkg_file=apt_detect + else + pkg_file=yum_detect + fi + + if ! crontab -l | grep -q "$pkg_file.sh"; then + echo -e "$(crontab -u root -l)\n*/5 * * * * /usr/local/bin/$pkg_file.sh > /var/lib/node_exporter/$pkg_file.prom 2>&1" | crontab -u root - + fi + + ##################### + ### Create Script ### + ##################### + if [ "$OS" = ubuntu ]; then + touch /usr/local/bin/apt_detect.sh + + { + echo '#!/bin/bash' + echo '' + echo '#####################################################' + echo '### ###' + echo '### Description: Expose metrics from apt updates. ###' + echo '### ###' + echo '### Phil Connor pconnor@ara.com ###' + echo '### Version 1.2.2.031524 ###' + echo '### ###' + echo '#####################################################' + # shellcheck disable=SC1003,SC2016 + echo 'upgrades="$(/usr/bin/apt-get --just-print upgrade \' + # shellcheck disable=SC1003 + echo ' | /usr/bin/awk -F'\''[()]'\'' \' + # shellcheck disable=SC2016 + echo ' '\''/^Inst/ { sub("^[^ ]+ ", "", $2); gsub(" ","",$2);' + # shellcheck disable=SC1003,SC2016,SC2028 + echo ' sub("\\[", " ", $2); sub("\\]", "", $2); print $2 }'\'' \' + # shellcheck disable=SC1003 + echo ' | /usr/bin/sort \' + # shellcheck disable=SC1003 + echo ' | /usr/bin/uniq -c \' + # shellcheck disable=SC2016,SC2028 + echo ' | awk '\''{ gsub(/\\\\/, "\\\\", $2); gsub(/\\"/, "\\\"", $2);' + # shellcheck disable=SC2016 + echo ' gsub(/\[/, "", $3); gsub(/\]/, "", $3);' + # shellcheck disable=SC2016 + echo ' print "apt_upgrades_pending{origin=\"" $2 "\",arch=\"" $3 "\"} " $1}'\'' ' + echo ')"' + echo '' + # shellcheck disable=SC2016 + echo 'upgradelist=$(/usr/bin/apt-get --just-print upgrade | grep Inst | awk '\''{gsub(/\(|\)/,"",$4)}1 {gsub(/:/,".",$4)}1 \ ' + # shellcheck disable=SC2016 + echo '{gsub(/\[|\]/,"",$3)}1 {gsub(/:/," ",$5)}1 {print "apt_upgradelist{pkgname=\"" $2 "\",uvers=\"" $4 "\", overs=\"" $3 "\", repo=\"" $5 "\"} " 1}'\'')' + echo '' + echo 'echo '\''# HELP apt_upgrades_pending Apt package pending updates by origin.'\'' ' + echo 'echo '\''# TYPE apt_upgrades_pending gauge'\'' ' + # shellcheck disable=SC2016 + echo 'if [[ -n "${upgrades}" ]] ; then' + # shellcheck disable=SC2016 + echo ' echo "${upgrades}"' + echo 'else' + echo ' echo '\''apt_upgrades_pending{origin="",arch=""} 0'\'' ' + echo 'fi' + echo '' + echo 'echo '\''# HELP apt_upgradelist List of packages for upgrade'\'' ' + echo 'echo '\''# TYPE apt_upgradelist gauge'\'' ' + # shellcheck disable=SC2016 + echo 'if [[ -n "${upgradelist}" ]]; then' + # shellcheck disable=SC2016 + echo ' echo "${upgradelist}"' + echo 'else' + echo ' echo '\''apt_upgradelist{origin="",arch=""} 0'\'' ' + echo 'fi' + echo 'echo '\''# HELP node_reboot_required Node reboot is required for software updates.'\'' ' + echo '# TYPE node_reboot_required gauge' + echo 'if [[ -f '/run/reboot-required' ]] ; then' + echo ' echo '\''node_reboot_required 1'\'' ' + echo 'else' + echo ' echo '\''node_reboot_required 0'\'' ' + echo 'fi' + } > /usr/local/bin/apt_detect.sh + + chmod 755 /usr/local/bin/apt_detect.sh + else + touch /usr/local/bin/yum_detect.sh + + { + echo '#! /bin/bash' + echo '' + echo '#####################################################' + echo '### ###' + echo '### Description: Expose metrics from yum updates. ###' + echo '### ###' + echo '### Phil Connor pconnor@ara.com ###' + echo '### Version 1.1.5.031524 ###' + echo '### ###' + echo '#####################################################' + echo 'set -u -o pipefail' + echo '' + echo '# shellcheck disable=SC2016' + echo 'filter_awk_script='\'' ' + echo 'BEGIN { mute=1 }' + echo '/Obsoleting Packages/ {' + echo ' mute=0' + echo '}' + echo 'mute && /^[[:print:]]+\.[[:print:]]+/ {' + # shellcheck disable=SC2016 + echo ' print $3' + echo '}' + echo ''\'' ' + echo '' + echo 'check_upgrades() {' + echo ' {' + echo ' /usr/bin/yum -q check-update |' + echo ' /usr/bin/xargs -n3 |' + # shellcheck disable=SC2016 + echo ' awk "${filter_awk_script}" |' + echo ' sort |' + echo ' uniq -c |' + # shellcheck disable=SC2016 + echo ' awk '\''{print "yum_upgrades_pending{origin=\""$2"\"} "$1}'\'' ' + echo ' }' + echo '}' + echo '' + # shellcheck disable=SC2016 + echo 'upgrades=$(check_upgrades)' + echo '' + echo 'echo '\''# HELP yum_upgrades_pending Yum package pending updates by origin.'\'' ' + echo 'echo '\''# TYPE yum_upgrades_pending gauge'\'' ' + # shellcheck disable=SC2016 + echo 'if [[ -n "${upgrades}" ]]; then' + # shellcheck disable=SC2016 + echo ' echo "${upgrades}"' + echo 'else' + echo ' echo '\''yum_upgrades_pending{origin=""} 0'\'' ' + echo 'fi' + echo '' + # shellcheck disable=SC2016 + echo 'upgradelist=$(yum check-update | awk '\''NR>4 {print "yum_upgradelist{pkgname=\"" $1 "\", uvers=\"" $2 "\", repo=\"" $3 "\"}", 1}'\'')' + echo '' + echo 'echo '\''# HELP yum_upgradelist List of packages for upgrade'\'' ' + echo 'echo '\''# TYPE yum_upgradelist gauge'\'' ' + # shellcheck disable=SC2016 + echo 'if [[ -n "${upgradelist}" ]]; then' + # shellcheck disable=SC2016 + echo ' echo "${upgradelist}"' + echo 'else' + echo ' echo '\''yum_upgradelist{pkgname=""} 0'\'' ' + echo 'fi' + echo '' + echo '### If yum-utils/dnf-utils is not installed this will skip reboot required metric' + echo 'if [[ -x /bin/needs-restarting ]]; then' + echo ' echo '\''# HELP node_reboot_required Node reboot is required for software updates.'\'' ' + echo ' echo '\''# TYPE node_reboot_required gauge'\'' ' + echo ' if /bin/needs-restarting -r > /dev/null 2>&1; then' + echo ' echo '\''node_reboot_required 0'\'' ' + echo ' else' + echo ' echo '\''node_reboot_required 1'\'' ' + echo ' fi' + echo 'fi' + } > /usr/local/bin/yum_detect.sh + + chmod 755 /usr/local/bin/yum_detect.sh + fi + } +} + +install_salt_status() { + { + if [ "$(command -v /bin/ss)" ]; then + ssdir=/bin + elif [ "$(command -v /usr/bin/ss)" ]; then + ssdir=/usr/bin + else + ssdir=/usr/sbin + fi + + ############################ + #### Check for Cron Job #### + ############################ + if ! crontab -l | grep -q "salt_status.sh"; then + echo -e "$(crontab -u root -l)\n*/5 * * * * /usr/local/bin/salt_status.sh > /var/lib/node_exporter/salt_status.prom 2>&1" | crontab -u root - + fi + + ##################### + ### Create Script ### + ##################### + touch /usr/local/bin/salt_status.sh + + { + echo '#! /bin/bash' + echo '#####################################################' + echo '### ###' + echo '### Description: Expose metrics from salt-minion. ###' + echo '### ###' + echo '### Phil Connor pconnor@ara.com ###' + echo '### Version 1.3.3.122023 ###' + echo '### ###' + echo '#####################################################' + echo '' + echo '## Local Command Variables' + # shellcheck disable=SC2016 + echo 'if [ ! "$(command -v salt-call)" ]; then' + echo ' saltck=none' + echo 'else' + # shellcheck disable=SC2016,SC2028 + echo ' saltck=$(salt-call test.ping | grep '\''\bTrue\b'\'')' + echo 'fi' + echo '' + # shellcheck disable=SC2016,SC2028 + echo "status=\$($ssdir/ss -nt | grep '\b4505\b')" + echo '' + echo '## Check If minion is connected to port 4505' + echo 'echo '\''# HELP minion_connection_status Shows if Salt-Minion is connected to Salt-Master.'\'' ' + echo 'echo '\''# TYPE minion_connection_status gauge'\'' ' + # shellcheck disable=SC2016 + echo 'if [[ -n "${status}" ]]; then' + echo ' echo '\''minion_connection_status{status=""} 1'\'' ' + echo 'else' + echo ' echo '\''minion_connection_status{status=""} 0'\'' ' + echo 'fi' + echo '' + echo '## Check to see in minion can ping master' + echo 'echo '\''# HELP minion_ping_status Shows if Salt-Minion is able to ping Salt-Master.'\'' ' + echo 'echo '\''# TYPE minion_ping_status gauge'\'' ' + # shellcheck disable=SC2016 + echo 'if [ $saltck = "error" ]; then' + echo ' echo '\''minion_ping_status{status=""} 3'\'' ' + # shellcheck disable=SC2016 + echo 'elif [ $saltck = none ]; then' + echo ' echo '\''minion_ping_status{status=""} 2'\'' ' + # shellcheck disable=SC2016 + echo 'elif [[ -n "${saltck}" ]]; then' + echo ' echo '\''minion_ping_status{status=""} 1'\'' ' + echo 'else' + echo ' echo '\''minion_ping_status{status=""} 0'\'' ' + echo 'fi' + } > /usr/local/bin/salt_status.sh + + chmod 755 /usr/local/bin/salt_status.sh + } +} + +install_process_list() { + { + ############################ + #### Check for Cron Job #### + ############################ + if ! crontab -l | grep -q "process_list.sh"; then + echo -e "$(crontab -u root -l)\n*/3 * * * * /usr/local/bin/process_list.sh > /var/lib/node_exporter/process_list.prom 2>&1" | crontab -u root - + fi + + ################################## + ### Check if lsof is installed ### + ################################## + if [ ! "$(command -v lsof)" ]; then + $pkgmgr install lsof + fi + + ########################## + ### Set lsof Dirictory ### + ########################## + if [ "$(command -v /bin/lsof)" ]; then + lfdir=/bin + elif [ "$(command -v /usr/bin/lsof)" ]; then + lfdir=/usr/bin + else + lfdir=/usr/sbin + fi + + ##################### + ### Create Script ### + ##################### + touch /usr/local/bin/process_list.sh + + { + echo '#! /bin/bash' + echo '' + echo '########################################################' + echo '### ###' + echo '### Description: Expose metrics from cpu by process. ###' + echo '### ###' + echo '### Phil Connor pconnor@ara.com ###' + echo '### Version 2.8.1.031424 ###' + echo '### ###' + echo '########################################################' + echo '' + echo '#############################' + echo '### Process List Function ###' + echo '#############################' + echo 'processes_list() {' + echo ' {' + # shellcheck disable=SC2016 + echo ' PList=$(ps aux)' + echo '' + echo ' while read -r PList' + echo ' do' + # shellcheck disable=SC2016 + echo ' pl=$(/usr/bin/awk '\''{print "node_cpu_usage{process=\""$11"\", pid=\""$2"\", owner=\""$1"\"}", $3}'\'')' + # shellcheck disable=SC2016 + echo ' done <<< "$PList"' + echo '' + echo ' echo '\''# HELP node_cpu_usage Usage of CPU by process.'\'' ' + echo ' echo '\''# TYPE node_cpu_usage gauge'\'' ' + # shellcheck disable=SC2016 + echo ' echo "$pl"' + echo ' }' + echo '}' + echo '' + echo '##################################' + echo '### File Handler List Function ###' + echo '##################################' + echo 'filehandlers_list() {' + echo ' {' + # shellcheck disable=SC2016,SC2028 + echo " Fhlist=\$($lfdir/lsof | /usr/bin/awk '{gsub(/\\\\/,\"\",\$1)}1 {gsub(/\\\\|\:/,\"\",\$4)}1 {print \$1 \" \" \$2 \" \" \$4}' | sort | uniq -c | sort -rn | head -20)" + echo '' + echo ' while read -r Fhlist' + echo ' do' + # shellcheck disable=SC2016 + echo ' fhl=$(/usr/bin/awk '\''{print "node_file_handlers{pid=\""$1"\", program=\""$2"\"}", $3}'\'')' + # shellcheck disable=SC2016 + echo ' done <<< "$Fhlist"' + echo ' echo '\''# HELP node_file_handler Usage of File Handlers.'\'' ' + echo ' echo '\''# TYPE node_file_handler gauge'\'' ' + # shellcheck disable=SC2016 + echo ' echo "$fhl"' + echo '' + # shellcheck disable=SC2016 + echo ' fhmax=$(cat /proc/sys/fs/file-nr | awk '\''{print $3}'\'')' + # shellcheck disable=SC2016 + echo ' fhfree=$(cat /proc/sys/fs/file-nr | awk '\''{print $2}'\'')' + # shellcheck disable=SC2016 + echo ' fhopen=$(cat /proc/sys/fs/file-nr | awk '\''{print $1}'\'')' + echo ' echo '\''# HELP node_max_files Max File Limit Handlers.'\'' ' + echo ' echo '\''# TYPE node_max_files gauge'\'' ' + # shellcheck disable=SC2016 + echo ' echo "node_total_max_files_handles $fhmax"' + # shellcheck disable=SC2016 + echo ' echo "node_total_free_file_handles $fhfree"' + # shellcheck disable=SC2016 + echo ' echo "node_total_open_file_handles $fhopen"' + echo '' + echo ' }' + echo '}' + echo '' + echo '################################' + echo '### File Handler Connections ###' + echo '################################' + echo 'filehandler_connections() {' + echo ' {' + # shellcheck disable=SC2016,SC2028 + echo " fhconn=\$($lfdir/lsof -i | /usr/bin/awk '{gsub(/\\\\/,\"_\",\$1)}1 {gsub(/\\*|\\:/\"_\",\$2)}1 {gsub(/\\*|\\:|\\\\/,\"\",\$9)}1')" + echo '' + echo ' while read -r fhconn' + echo ' do' + # shellcheck disable=SC2016 + echo ' conn=$(/usr/bin/awk '\''NR>1 {print "node_file_handle_connection{command=\""$1"\", connection=\""$9"\", user=\""$3"\", protocol=\""$5"\", type=\""$8"\"}", $2}'\'')' + # shellcheck disable=SC2016 + echo ' done <<< "$fhconn"' + echo ' echo '\''# HELP node_file_handle_connection Connections by process.'\'' ' + echo ' echo '\''# TYPE node_file_handle_connection gauge'\'' ' + # shellcheck disable=SC2016 + echo ' echo "$conn"' + echo ' }' + echo '}' + echo '' + echo '######################' + echo '### Function Calls ###' + echo '######################' + echo 'processes_list' + echo 'filehandlers_list' + echo 'filehandler_connections' + } > /usr/local/bin/process_list.sh + + chmod 755 /usr/local/bin/process_list.sh + + } +} +######################## +#### Function Calls #### +######################## +install_node_exporter +#install_loki +#install_directory_size +install_apt_yum_detect +install_salt_status +install_process_list \ No newline at end of file diff --git a/processes_usage.sh b/processes_usage.sh new file mode 100644 index 0000000..b2529d9 --- /dev/null +++ b/processes_usage.sh @@ -0,0 +1,161 @@ +#! /bin/bash + +######################## +### System Variables ### +######################## +if [ "$(command -v lsb_release)" ]; then + OS=$(lsb_release -i | awk '{print $3}' | tr '[:upper:]' '[:lower:]') + OSVER=$(lsb_release -r | awk '{print $2}' | awk -F. '{print $1}') +else + OS=$(grep PRETTY_NAME /etc/os-release | sed 's/PRETTY_NAME=//g' | tr -d '="' | awk '{print $1}' | tr '[:upper:]' '[:lower:]') + OSVER=$(grep VERSION_ID /etc/os-release | sed 's/VERSION_ID=//g' | tr -d '="' | awk -F. '{print $1}') +fi + +########################################################### +#### Detect Package Manger from OS and OSVer Variables #### +########################################################### +if [[ "${OS}" = amazon || "${OS}" = red ]]; then + if [[ "${OSVER}" = 2 || "${OSVER}" = 7 ]]; then + pkgmgr="yum -y" + else + pkgmgr="dnf -y" + fi +elif [ "${OS}" = ubuntu ]; then + pkgmgr="apt -y" +fi + +############################# +#### Check for Directory #### +############################# +if [ ! -d "/var/lib/node_exporter/" ]; then + mkdir -p /var/lib/node_exporter + chown prometheus. /var/lib/node_exporter +fi + +############################ +#### Check for Cron Job #### +############################ +if ! crontab -l | grep -q "cpu_usage.sh"; then + echo -e "$(crontab -u root -l)\n*/3 * * * * /usr/local/bin/cpu_usage.sh > /var/lib/node_exporter/cpu_usage.prom 2>&1" | crontab -u root - +fi + +if [ ! "$(command -v lsof)" ]; then + $pkgmgr install lsof +fi + +touch /usr/local/bin/cpu_usage.sh + +{ + echo '#! /bin/bash' + echo '' + echo '########################################################' + echo '### ###' + echo '### Description: Expose metrics from cpu by process. ###' + echo '### ###' + echo '### Phil Connor pconnor@ara.com ###' + echo '### Version 2.7.8.020524 ###' + echo '### ###' + echo '########################################################' + echo '' + echo '#############################' + echo '### Process List Function ###' + echo '#############################' + echo 'processes_list() {' + echo ' {' + # shellcheck disable=SC2016 + echo ' PList=$(ps aux)' + echo '' + echo ' while read -r PList' + echo ' do' + # shellcheck disable=SC2016 + echo ' pl=$(/usr/bin/awk '\''{print "node_cpu_usage{process=\""$11"\", pid=\""$2"\", owner=\""$1"\"}", $3}'\'')' + # shellcheck disable=SC2016 + echo ' done <<< "$PList"' + echo '' + echo ' echo '\''# HELP node_cpu_usage Usage of CPU by process.'\'' ' + echo ' echo '\''# TYPE node_cpu_usage gauge'\'' ' + # shellcheck disable=SC2016 + echo ' echo "$pl"' + echo ' }' + echo '}' + echo '' + echo '##################################' + echo '### File Handler List Function ###' + echo '##################################' + echo 'filehandlers_list() {' + echo ' {' + # shellcheck disable=SC2016,SC2028 + echo ' Fhlist=$(/usr/sbin/lsof | /usr/bin/awk '\''{gsub(/\\/,"",$1)}1 {gsub(/\\|\:/,"",$4)}1 {print $1 " " $2 " " $4}'\'' | sort | uniq -c | sort -rn | head -30)' + echo '' + echo ' while read -r Fhlist' + echo ' do' + # shellcheck disable=SC2016 + echo ' fhl=$(/usr/bin/awk '\''{print "node_file_handlers{pid=\""$1"\", program=\""$2"\"}", $3}'\'')' + # shellcheck disable=SC2016 + echo ' done <<< "$Fhlist"' + echo ' echo '\''# HELP node_file_handler Usage of File Handlers.'\'' ' + echo ' echo '\''# TYPE node_file_handler gauge'\'' ' + # shellcheck disable=SC2016 + echo ' echo "$fhl"' + echo '' + # shellcheck disable=SC2016 + echo ' fhmax=$(cat /proc/sys/fs/file-nr | awk '\''{print $3}'\'')' + # shellcheck disable=SC2016 + echo ' fhfree=$(cat /proc/sys/fs/file-nr | awk '\''{print $2}'\'')' + # shellcheck disable=SC2016 + echo ' fhopen=$(cat /proc/sys/fs/file-nr | awk '\''{print $1}'\'')' + echo ' echo '\''# HELP node_max_files Max File Limit Handlers.'\'' ' + echo ' echo '\''# TYPE node_max_files gauge'\'' ' + # shellcheck disable=SC2016 + echo ' echo "node_total_max_files_handles $fhmax"' + # shellcheck disable=SC2016 + echo ' echo "node_total_free_file_handles $fhfree"' + # shellcheck disable=SC2016 + echo ' echo "node_total_open_file_handles $fhopen"' + echo '' + echo ' }' + echo '}' + echo '' + echo '################################' + echo '### File Handler Connections ###' + echo '################################' + echo 'filehandler_connections() {' + echo ' {' + # shellcheck disable=SC2016,SC2028 + echo ' fhconn=$(/usr/sbin/lsof -i | /usr/bin/awk '\''{gsub(/\\/,"_",$1)}1 {gsub(/\*|\:/"_",$2)}1 {gsub(/\*|\:|\\/,"",$9)}1'\'')' + echo '' + echo ' while read -r fhconn' + echo ' do' + # shellcheck disable=SC2016 + echo ' conn=$(/usr/bin/awk '\''NR>1 {print "node_file_handle_connection{command=\""$1"\", connection=\""$9"\", user=\""$3"\", protocol=\""$5"\", type=\""$8"\"}", $2}'\'')' + # shellcheck disable=SC2016 + echo ' done <<< "$fhconn"' + echo ' echo '\''# HELP node_file_handle_connection Connections by process.'\'' ' + echo ' echo '\''# TYPE node_file_handle_connection gauge'\'' ' + # shellcheck disable=SC2016 + echo ' echo "$conn"' + echo ' }' + echo '}' + echo '' + echo '######################' + echo '### Function Calls ###' + echo '######################' + echo 'processes_list' + echo 'filehandlers_list' + echo 'filehandler_connections' +} > /usr/local/bin/cpu_usage.sh + +chmod 755 /usr/local/bin/cpu_usage.sh + + +################# +### SEARCH="java" +### for i in $(ps -C "${SEARCH}" -o pid | grep -v PID); do echo "PID # ${i} open files count : $(sudo ls -l /proc/${i}/fd | wc -l)"; done +############### + +############### +### fuser -vm / 2>&1 | awk '$3 ~ /f|F/' | while read user pid flags rest; do printf '%10s %10s %10s %s\n' $user $pid $flags "$( #### +############################################################# + +######################## +#### User Variables #### +######################## +domain=mylinux.work +email=phil@$domain +######################## +### System Variables ### +######################## +if [ "$(command -v lsb_release)" ]; then + OS=$(lsb_release -i | awk '{print $3}' | tr '[:upper:]' '[:lower:]') +else + OS=$(grep PRETTY_NAME /etc/os-release | sed 's/PRETTY_NAME=//g' | tr -d '="' | awk '{print $1}' | tr '[:upper:]' '[:lower:]') +fi + +PYAML=/etc/prometheus + +if ! [ -d "/usr/lib/systemd/system" ]; then + psdir='/etc/systemd/system' +else + psdir='/usr/lib/systemd/system' +fi + +######################### +### Check permissions ### +######################### +if [[ $EUID -ne 0 ]]; then + echo '' + echo "This script must be run as root! Login as root, or sudo/su." + echo '' + exit 1; +fi + +###################### +### Package Manager ## +###################### +if [ "$OS" = ubuntu ]; then + pkgmgr='apt -y' +else + pkgmgr='dnf -y' +fi + +################################### +#### Add Prometheus User/Group #### +################################### +if ! grep prometheus /etc/passwd; then + groupadd --system prometheus + if [ "$OS" = ubuntu ]; then + useradd -s /sbin/nologin --system -g prometheus prometheus + else + useradd -m -s /bin/false prometheus -g prometheus + fi +fi + +################################# +#### Check for wget and curl #### +################################# +if [ ! "$(command -v wget)" ]; then + $pkgmgr install wget +fi + +if [ ! "$(command -v curl)" ]; then + $pkgmgr install curl +fi + +########################## +### Install Prometheus ### +########################## +install_prometheus() { + { + mkdir /etc/prometheus + mkdir /var/lib/prometheus + chown prometheus /var/lib/prometheus/ + + for i in backups rules templates consoles console_libraries + do + mkdir -p /etc/prometheus/${i} + chown -R prometheus. /etc/prometheus/${i} + chmod -R 755 /etc/prometheus/${i} + done + + cd /tmp || exit 2 + curl -s https://api.github.com/repos/prometheus/prometheus/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi - + tar -xvf prometheus*.tar.gz + cd prometheus-*/ || exit 2 + mv prometheus promtool /usr/local/bin/ + chown prometheus. /usr/local/bin/prometheus /usr/local/bin/promtool + mv prometheus.yml /etc/prometheus/ + mv consoles/ console_libraries/ /etc/prometheus/ + chown -R prometheus. /var/lib/prometheus/ + + if [ "$OS" = red ]; then + if [ "$OSVER" = 8 ]; then + restorecon -rv /usr/local/bin/node_exporter + fi + fi + + cp $PYAML/prometeus.yml $PYAML/backups/ + { + echo '# Global config' + echo 'global:' + echo ' scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. ' + echo ' evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. ' + echo ' scrape_timeout: 15s # scrape_timeout is set to the global default (10s).' + echo '' + echo '# Alertmanager configuration' + echo 'alerting:' + echo ' alertmanagers:' + echo ' - static_configs:' + echo ' - targets:' + echo ' - alertmanager:9093' + echo '' + echo '# Load rules once and periodically evaluate them according to the global '\''evaluation_interval'\''.' + echo 'rule_files:' + echo '# - "first_rules.yml"' + echo '# - "second_rules.yml"' + echo '' + echo '# A scrape configuration containing exactly one endpoint to scrape:# Here it'\''s Prometheus itself.' + echo 'scrape_configs:' + echo ' # The job name is added as a label '\''job='\'' to any timeseries scraped from this config.' + echo ' - job_name: '\''prometheus'\''' + echo '' + echo ' # metrics_path defaults to '/metrics'' + echo ' # scheme defaults to '\''http'\''.' + echo '' + echo ' static_configs:' + echo ' - targets: ['\''localhost:9090'\'']' + echo '' + echo ' - job_name: '\''server_metrics'\''' + echo ' scrape_interval: 5s' + echo ' static_configs:' + echo ' - targets: ['\''localhost:9100'\'']' + echo ' labels:' + echo ' alias: Prometheus Server' + } > /etc/prometheus/prometheus.yml + + ### Not required used for my test machine ### + # firewall-cmd --add-port=9090/tcp + # firewall-cmd --add-port=9090/tcp --permanent + # + + touch $psdir/prometheus.service + + { + echo '[Unit]' + echo 'Description=Prometheus Time Series Collection and Processing Server' + echo 'Documentation=https://prometheus.io/docs/introduction/overview/' + echo 'Wants=network-online.target' + echo 'After=network-online.target' + echo '' + echo '[Service]' + echo 'Type=simple' + echo 'User=prometheus' + echo 'Group=prometheus' + echo '' + echo "ExecReload=/bin/kill -HUP \$MAINPID" + echo "ExecStart=/usr/local/bin/prometheus \\" + echo " --config.file /etc/prometheus/prometheus.yml \\" + echo " --storage.tsdb.path /var/lib/prometheus/data \\" + echo " --web.console.templates=/etc/prometheus/consoles \\" + echo " --web.console.libraries=/etc/prometheus/console_libraries \\" + echo " --web.listen-address=0.0.0.0:9090 \\" + echo " --web.external-url= \\" + echo ' --enable-feature=new-service-discovery-manager,exemplar-storage,extra-scrape-metrics' + echo '' + echo 'Restart=always' + echo 'RestartSec=5s' + if [ "$OS" = ubuntu ]; then + echo 'SyslogIdentifier=prometheus' + fi + echo 'Restart=always' + echo '' + echo '[Install]' + echo 'WantedBy=multi-user.target' + } > $psdir/prometheus.service + + systemctl daemon-reload + systemctl enable --now prometheus + + } +} + +############################# +### Install node_exporter ### +############################# +install_node_exporter() { + { + cd /tmp || exit 2 + curl -s https://api.github.com/repos/prometheus/node_exporter/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi - + tar -xvf node_exporter*.tar.gz + cd node_exporter-*/ || exit 2 + mv node_exporter /usr/local/bin + chown prometheus. /usr/local/bin/node_exporter + + if [ "$OS" = red ]; then + if [ "$OSVER" = 8 ]; then + restorecon -rv /usr/local/bin/node_exporter + fi + fi + + touch $psdir/node_exporter.service + { + echo '[Unit]' + echo 'Description=Prometheus Node Exporter' + echo 'Wants=network-online.target' + echo 'After=network-online.target' + echo '' + echo '[Service]' + echo 'User=prometheus' + echo 'Group=prometheus' + echo 'Type=simple' + echo "ExecStart=/usr/local/bin/node_exporter \\" + echo " --collector.ethtool \\" + echo " --collector.interrupts \\" + echo " --collector.processes \\" + echo " --collector.systemd \\" + echo ' --collector.tcpstat' + echo '' + echo '[Install]' + echo 'WantedBy=multi-user.target' + } > $psdir/node_exporter.service + + systemctl daemon-reload + systemctl enable --now node_exporter + + ### Not required used for my test machine ### + # firewall-cmd --add-port=9100/tcp + # firewall-cmd --add-port=9100/tcp --permanent + + systemctl restart prometheus + } +} + +######################## +### Install BlackBox ### +######################## +install_blackbox() { + { + cd /tmp || exit 2 + curl -s https://api.github.com/repos/prometheus/blackbox_exporter/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi - + tar -xvf blackbox_exporter*.tar.gz + cd blackbox_exporter-*/ || exit 2 + mv blackbox_exporter /usr/local/bin + chown prometheus. /usr/local/bin/blackbox_exporter + mv blackbox.yml /etc/prometheus/ + chown -R prometheus. /etc/prometheus/ + + if [ "$OS" = red ]; then + if [ "$OSVER" = 8 ]; then + restorecon -rv /usr/local/bin/node_exporter + fi + fi + + touch $psdir/blackbox_exporter.service + + { + echo '[Unit]' + echo 'Description=Prometheus Blackbox Exporter Http/Https Montoring' + echo 'After=network.target' + echo '' + echo '[Service]' + echo 'User=prometheus' + echo 'Group=prometheus' + echo 'Type=simple' + echo "ExecStart=/usr/local/bin/blackbox_exporter \\" + echo " --config.file /etc/prometheus/blackbox.yml \\" + echo ' --web.listen-address=":9115"' + echo '' + echo 'Restart=always' + echo '' + echo '[Install]' + echo 'WantedBy=multi-user.target' + } > $psdir/blackbox_exporter.service + + systemctl daemon-reload + systemctl enable --now blackbox_exporter + + { + echo ' - job_name: '"'blackbox'"'' + echo ' metrics_path: /probe' + echo ' params:' + echo ' module: [http_2xx]' + echo ' static_configs:' + echo ' - targets:' + echo ' #### Local Targets ####' + echo ' - http://localhost:9090' + echo '' + echo ' #### Remote Targets ####' + echo ' #- https://google.com' + echo '' + echo ' relabel_configs:' + echo ' - source_labels: [__address__]' + echo ' target_label: __param_target' + echo ' - source_labels: [__param_target]' + echo ' target_label: instance' + echo ' - target_label: __address__' + echo ' replacement: localhost:9115' + } >> $PYAML/prometheus.yml + + cp $PYAML/blackbox.yml $PYAML/backups/ + + { + echo 'modules:' + echo ' http_2xx:' + echo ' prober: http' + echo ' timeout: 20s' + echo ' http:' + echo ' valid_status_codes: []' + echo ' valid_http_versions: ["HTTP/1.1", "HTTP/2"]' + echo ' ip_protocol_fallback: false' + echo ' method: GET' + echo ' follow_redirects: true' + echo ' preferred_ip_protocol: "ip4"' + echo ' fail_if_ssl: false' + echo ' fail_if_not_ssl: false' + echo ' tls_config:' + echo ' insecure_skip_verify: true' + echo ' #basic_auth:' + echo ' #username: "username"' + echo ' #password: "password"' + echo ' http_post_2xx:' + echo ' prober: http' + echo ' http:' + echo ' method: POST' + echo ' tcp_connect:' + echo ' prober: tcp' + echo ' pop3s_banner:' + echo ' prober: tcp' + echo ' tcp:' + echo ' query_response:' + echo ' - expect: "^+OK"' + echo ' tls: true' + echo ' tls_config:' + echo ' insecure_skip_verify: false' + echo ' grpc:' + echo ' prober: grpc' + echo ' grpc:' + echo ' tls: true' + echo ' preferred_ip_protocol: "ip4"' + echo ' grpc_plain:' + echo ' prober: grpc' + echo ' grpc:' + echo ' tls: false' + echo ' service: "service1"' + echo ' ssh_banner:' + echo ' prober: tcp' + echo ' tcp:' + echo ' query_response:' + echo ' - expect: "^SSH-2.0-"' + echo ' - send: "SSH-2.0-blackbox-ssh-check"' + echo ' irc_banner:' + echo ' prober: tcp' + echo ' tcp:' + echo ' query_response:' + echo ' - send: "NICK prober"' + echo ' - send: "USER prober prober prober :prober"' + echo ' - expect: "PING :([^ ]+)"' + echo ' send: "PONG '\$'{1}"' + echo ' - expect: "^:[^ ]+ 001"' + echo ' icmp:' + echo ' prober: icmp' + echo ' icmp_ttl5:' + echo ' prober: icmp' + echo ' timeout: 5s' + echo ' icmp:' + echo ' ttl: 5' + } > $PYAML/blackbox.yml + + if ! blackbox_exporter --config.check --config.file $PYAML/blackbox.yml; then + echo "There's a problem with the blackbox_exporter.yml config file" + exit 2 + else + systemctl restart blackbox_exporter + fi + + ### Not required used for my test machine ### + # firewall-cmd --add-port=9115/tcp + # firewall-cmd --add-port=9115/tcp --permanent + } +} + +############################ +### Install AlertManager ### +############################ +install_alertmanager() { + { + cd /tmp || exit 2 + curl -s https://api.github.com/repos/prometheus/alertmanager/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi - + tar -xvf alertmanager*.tar.gz + cd alertmanager-*/ || exit 2 + mv amtool alertmanager /usr/local/bin + mv alertmanager.yml /etc/prometheus + chown -R prometheus. /etc/prometheus + chown prometheus. /usr/local/bin/alertmanager /usr/local/bin/amtool + mkdir /var/lib/alertmanager + chown prometheus. /var/lib/alertmanager + + if [ "$OS" = red ]; then + if [ "$OSVER" = 8 ]; then + restorecon -rv /usr/local/bin/alertmanager + fi + fi + + touch $psdir/alertmanager.service + + { + echo '[Unit]' + echo 'Description=Prometheus AlertManager Service' + echo 'Wants=network-online.target' + echo 'After=network-online.target' + echo '' + echo '[Service]' + echo 'User=prometheus' + echo 'Group=prometheus' + echo 'Type=simple' + echo "ExecStart=/usr/local/bin/alertmanager \\" + echo " --config.file /etc/prometheus/alertmanager.yml \\" + echo " --storage.path /var/lib/alertmanager/ \\" + echo ' --cluster.advertise-address=0.0.0.0:9093' + echo '' + echo '[Install]' + echo 'WantedBy=multi-user.target' + } > $psdir/alertmanager.service + + cp $PYAML/alertmanager.yml $PYAML/backups/ + + { + echo 'global:' + echo ' smtp_smarthost: '\''nlb-exch-imaps-smt-x28bvtlo2e4rs-60fe7830ff427ab2.elb.us-gov-west-1.amazonaws.com:25'\''' + echo ' smtp_from: '\''alertmanager@calormen.net'\''' + echo ' smtp_require_tls: false' + echo '' + echo 'templates:' + echo ' - '\''/etc/prometheus/templates/email_alert.tmpl'\''' + echo '' + echo 'route:' + echo ' receiver: email' + echo '' + echo 'receivers:' + echo ' - name: '\''email'\''' + echo ' email_configs:' + echo " - to: '$email'" + echo ' from: '\''prometheus@us.calormen.net'\''' + echo ' #html: '\''{{ template "email" .}}'\''' + echo ' send_resolved: true' + } > $PYAML/alertmanager.yml + + touch $PYAML/templates/email_alerts.tmpl + + { + echo '{{ define "email" }}' + echo '' + echo '' + echo ' ' + echo ' ' + echo ' ' + echo ' ' + echo ' ' + echo ' ' + echo ' ' + echo ' ' + echo ' ' + echo ' ' + echo ' ' + echo ' ' + echo ' ' + echo '' + echo ' ' + echo ' {{ range .Alerts }}' + echo ' ' + echo ' ' + echo ' ' + echo ' ' + echo ' ' + echo ' ' + echo ' {{ end }}' + echo ' ' + echo '' + echo '
Alert nameHostSummaryDescription
{{ .Labels.alertname }}{{ .Annotations.host }}{{ .Annotations.summary }}{{ .Annotations.description }}
' + echo ' ' + echo '' + echo '' + echo '{{end}}' + } > $PYAML/templates/email_alerts.tmpl + + systemctl daemon-reload + systemctl --now enable alertmanager + + } +} + +####################### +### Install Grafana ### +####################### +install_grafana() { + { + if [ "$OS" = ubuntu ]; then + $pkgmgr install -y apt-transport-https software-properties-common + mkdir -p /etc/apt/keyrings/ + wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/grafana.gpg > /dev/null + echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" | sudo tee -a /etc/apt/sources.list.d/grafana.list + $pkgmgr update + $pkgmgr install grafana + else + touch /etc/yum.repos.d/grafana.repo + { + echo '[grafana]' + echo 'name=grafana' + echo 'baseurl=https://packages.grafana.com/oss/rpm' + echo 'repo_gpgcheck=1' + echo 'enabled=1' + echo 'gpgcheck=1' + echo 'gpgkey=https://packages.grafana.com/gpg.key' + echo 'sslverify=1' + echo 'sslcacert=/etc/pki/tls/certs/ca-bundle.crt' + } > /etc/yum.repos.d/grafana.repo + + dnf -y repolist + dnf -y install grafana + fi + + systemctl daemon-reload + systemctl enable --now grafana-server + + ### Not required used for my test machine ### + # firewall-cmd --add-port=3000/tcp + # firewall-cmd --add-port=3000/tcp --permanent + } +} + +install_nginx() { + { + $pkgmgr install nginx + sitesa=/etc/nginx/sites-available + sitese=/etc/nginx/sites-enabled/ + + touch $sitesa/prometheus.conf + { + echo 'server {' + echo ' listen 80;' + echo ' listen [::]:80;' + echo '' + echo " server_name prometheus.$domain;" + echo '' + echo ' location / {' + echo ' proxy_pass http://localhost:9090/;' + echo ' include proxy_params;' + echo ' }' + echo '}' + } > $sitesa/prometheus.conf + + touch $sitesa/metrics.conf + { + echo 'server {' + echo ' listen 80;' + echo ' listen [::]:80;' + echo '' + echo " server_name metrics.$domain;" + echo '' + echo ' location / {' + echo ' proxy_pass http://localhost:3000/;' + echo ' include proxy_params;' + echo ' }' + echo '}' + } > $sitesa/metrics.conf + + touch $sitesa/alerts.conf + { + echo 'server {' + echo ' listen 80;' + echo ' listen [::]:80;' + echo '' + echo " server_name alerts.$domain;" + echo '' + echo ' location / {' + echo ' proxy_pass http://localhost:9093/;' + echo ' include proxy_params;' + echo ' }' + echo '}' + } > $sitesa/alerts.conf + + touch $sitesa/loki.conf + { + echo 'server {' + echo ' listen 80;' + echo ' listen [::]:80;' + echo '' + echo " server_name loki.$domain;" + echo '' + echo ' location / {' + echo ' proxy_pass http://localhost:9093/;' + echo ' include proxy_params;' + echo ' }' + echo '}' + } > $sitesa/alerts.conf + + ln -s $sitesa/prometheus.conf $sitese + ln -s $sitesa/metrics.conf $sitese + ln -s $sitesa/alerts.conf $sitese + ln -s $sitesa/prometheus.conf $sitese + + systemctl restart nginx + } +} + +install_certbot() { + { + systemctl stop nginx + if [ "$OS" = ubuntu ]; then + $pkgmgr update + $pkgmgr install python3-certbot-nginx certbot + systemctl enable --now certbot.timer + else + $pkgmgr install epel-release + $pkgmgr install certbot python3-certbot-nginx + touch /etc/cron.weekly/certbot + chmod +x /etc/cron.weekly/certbot + { + echo '#!/bin/sh' + echo 'certbot renew' + } > /etc/cron.weekly/certbot + fi + } +} + +###################### +### Function Calls ### +###################### +install_prometheus +install_node_exporter +install_blackbox +install_alertmanager +install_grafana +install_nginx +#install_certbot \ No newline at end of file diff --git a/salt_status.sh b/salt_status.sh new file mode 100644 index 0000000..a82f5a2 --- /dev/null +++ b/salt_status.sh @@ -0,0 +1,73 @@ +#! /bin/bash + +if ! crontab -l | grep -q "salt_status.sh"; then + echo -e "$(crontab -u root -l)\n*/5 * * * * /usr/local/bin/salt_status.sh > /var/lib/node_exporter/salt_status.prom 2>&1" | crontab -u root - +fi + +if [ "$(command -v /bin/ss)" ]; then + dir=/bin +elif [ "$(command -v /usr/bin/ss)" ]; then + dir=/usr/bin +else + dir=/usr/sbin +fi + +install_salt_status() { + { + touch /usr/local/bin/salt_status.sh + + { + echo '#! /bin/bash' + echo '#####################################################' + echo '### ###' + echo '### Description: Expose metrics from salt-minion. ###' + echo '### ###' + echo '### Phil Connor contact@mylinux.work ###' + echo '### Version 1.3.3.122023 ###' + echo '### ###' + echo '#####################################################' + echo '' + echo '## Local Command Variables' + # shellcheck disable=SC2016 + echo 'if [ ! "$(command -v salt-call)" ]; then' + echo ' cert=none' + echo 'else' + # shellcheck disable=SC2016,SC2028 + echo ' cert=$(salt-call test.ping | grep '\''\bTrue\b'\'')' + echo 'fi' + echo '' + # shellcheck disable=SC2016,SC2028 + echo "status=\$($dir/ss -nt | grep '\b4505\b')" + echo '' + echo '## Check If minion is connected to port 4505' + echo 'echo '\''# HELP minion_connection_status Shows if Salt-Minion is connected to Salt-Master.'\'' ' + echo 'echo '\''# TYPE minion_connection_status gauge'\'' ' + # shellcheck disable=SC2016 + echo 'if [[ -n "${status}" ]]; then' + echo ' echo '\''minion_connection_status{status=""} 1'\'' ' + echo 'else' + echo ' echo '\''minion_connection_status{status=""} 0'\'' ' + echo 'fi' + echo '' + echo '## Check to see in minion can ping master' + echo 'echo '\''# HELP minion_ping_status Shows if Salt-Minion is able to ping Salt-Master.'\'' ' + echo 'echo '\''# TYPE minion_ping_status gauge'\'' ' + # shellcheck disable=SC2016 + echo 'if [ $cert = none ]; then' + echo ' echo '\''minion_ping_status{status=""} 2'\'' ' + # shellcheck disable=SC2016 + echo 'elif [[ -n "${cert}" ]]; then' + echo ' echo '\''minion_ping_status{status=""} 1'\'' ' + echo 'else' + echo ' echo '\''minion_ping_status{status=""} 0'\'' ' + echo 'fi' + } > /usr/local/bin/salt_status.sh + + chmod 755 /usr/local/bin/salt_status.sh + } +} + +######################## +#### Function Calls #### +######################## +install_salt_status \ No newline at end of file diff --git a/yum_detect.sh b/yum_detect.sh new file mode 100644 index 0000000..6c9dad7 --- /dev/null +++ b/yum_detect.sh @@ -0,0 +1,52 @@ +#! /bin/bash +##################################### +# Description: Expose metrics from yum updates. +# +# Phil Connor pconnor@ara.com +# Version 1.0.4.110523 +# Based on apt_detect.sh +###################################### +set -u -o pipefail + +# shellcheck disable=SC2016 +filter_awk_script=' +BEGIN { mute=1 } +/Obsoleting Packages/ { + mute=0 +} +mute && /^[[:print:]]+\.[[:print:]]+/ { + print $3 +} +' + +check_upgrades() { + { + /usr/bin/yum -q check-update | + /usr/bin/xargs -n3 | + awk "${filter_awk_script}" | + sort | + uniq -c | + awk '{print "yum_upgrades_pending{origin=\""$2"\"} "$1}' + } +} + +upgrades=$(check_upgrades) + +echo '# HELP yum_upgrades_pending Yum package pending updates by origin.' +echo '# TYPE yum_upgrades_pending gauge' +if [[ -n "${upgrades}" ]] ; then + echo "${upgrades}" +else + echo 'yum_upgrades_pending{origin=""} 0' +fi + +# If yum-utils/dnf-utils is not installed then we skip rendering this metric +if [[ -x /bin/needs-restarting ]] ; then + echo '# HELP node_reboot_required Node reboot is required for software updates.' + echo '# TYPE node_reboot_required gauge' + if /bin/needs-restarting -r > /dev/null 2>&1 ; then + echo 'node_reboot_required 0' + else + echo 'node_reboot_required 1' + fi +fi \ No newline at end of file