Initial commit

apt_detect.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+#########################################
+# Description: Expose metrics from apt updates.
+#
+# Phil Connor contact@mylinux.work
+# Version 1.0.1.103123
+#########################################
+
+upgrades="$(/usr/bin/apt-get --just-print upgrade | /usr/bin/awk -F'[()]' \
+    '/^Inst/ { sub("^[^ ]+ ", "", $2); gsub(" ","",$2); sub("\\[", " ", $2); sub("\\]", "", $2); print $2 }' \
+    | /usr/bin/sort | /usr/bin/uniq -c | awk '{ gsub(/\\\\/, "\\\\", $2); gsub(/\\"/, "\\\"", $2);
+        gsub(/\[/, "", $3); gsub(/\]/, "", $3);
+        print "apt_upgrades_pending{origin=\"" $2 "\",arch=\"" $3 "\"} " $1}'
+)"
+
+echo '# HELP apt_upgrades_pending Apt package pending updates by origin.'
+echo '# TYPE apt_upgrades_pending gauge'
+if [[ -n "${upgrades}" ]] ; then
+    echo "${upgrades}"
+else
+    echo 'apt_upgrades_pending{origin="",arch=""} 0'
+fi
+
+echo '# HELP node_reboot_required Node reboot is required for software updates.'
+echo '# TYPE node_reboot_required gauge'
+if [[ -f '/run/reboot-required' ]] ; then
+    echo 'node_reboot_required 1'
+else
+    echo 'node_reboot_required 0'
+fi

directory_size.sh

@@ -0,0 +1,35 @@
+#! /bin/bash
+###########################
+# Expose directory usage metrics, passed as an argument.
+#
+# Usage: add this to crontab:
+#
+# */5 * * * * directory_size.sh <directory_name> > /var/lib/node_exporter/directory_size.prom
+#
+# sed pattern taken from https://www.robustperception.io/monitoring-directory-sizes-with-the-textfile-collector/! 
+#
+# Phil Connor pconnor@ara.com
+# Version 1.0.3-111623
+###########################
+
+#############################
+#### Check for Directory ####
+#############################
+if [ ! -d "/var/lib/node_exporter/" ]; then
+    mkdir -p /var/lib/node_exporter
+    chown prometheus. /var/lib/node_exporter
+fi
+
+############################
+#### Check for Cron Job ####
+############################
+if ! crontab -l | grep -q 'directory_size.sh'; then
+    echo -e "$(crontab -u root -l)\n*/5 * * * * /usr/local/bin/directory_size.sh /var/log/audit > /var/lib/node_exporter/directory_size.prom 2>&1" | crontab -u root -
+fi
+
+###################################
+#### Directory Metrics Command ####
+###################################
+echo "# HELP node_directory_size_bytes Disk space used by some directories"
+echo "# TYPE node_directory_size_bytes gauge"
+du --block-size=1 --summarize "$@" | sed -ne 's/\\/\\\\/;s/"/\\"/g;s/^\([0-9]\+\)\t\(.*\)$/node_directory_size_bytes{directory="\2"} \1/p'

install_f2b_metrics.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+#############################################################
+#### Fail2Ban Metrics install Script for Prometheus      #### 
+#### Centos/Redhat and Debian/Ubuntu Servers.            ####       
+####                                                     ####
+#### Author: Phil Connor 08/27/2023                      ####
+#### Contact: contact@mylinux.work                       ####
+#### Version 1.92.011924                                 ####
+####                                                     ####
+#### To use this script chmod it to 755                  ####
+#### or simply type bash <filename.sh>                   ####  
+#############################################################
+
+
+
+
+if ! crontab -l | grep -q "fail2ban_metrics.sh"; then
+    echo -e "$(crontab -u root -l)\n*/5 * * * * /usr/local/bin/fail2ban_metrics.sh > /var/lib/node_exporter/fail2ban_metrics.prom 2>&1" | crontab -u root -
+fi
+
+install_fail2_metrics() {
+	{
+		{
+			echo 'jails=0'
+			echo ''
+			# shellcheck disable=SC2016,SC2028
+			echo 'for jail in $(fail2ban-client status | grep '\''Jail list:'\'' | sed '\''s/.*Jail list:[\t ]*//;s/,//g'\'')' 
+			echo '	do'
+			# shellcheck disable=SC2004,SC2016,SC2030
+			echo '		jails=$(($jails + 1))'
+			# shellcheck disable=SC2016
+			echo '		fail2ban-client status "$jail" | awk -F: '\'' '
+			# shellcheck disable=SC2016
+			echo '		$1 ~ /Currently failed/ {'
+			# shellcheck disable=SC2016,SC2154
+			echo '			print "fail2ban_failed_current{jail=\"'\''"$jail"'\''\"} " $2;'
+			echo '		}'
+			# shellcheck disable=SC2016
+			echo '		$1 ~ /Total failed/ {'
+			# shellcheck disable=SC2016
+			echo '			print "fail2ban_failed_total{jail=\"'\''"$jail"'\''\"} " $2;'
+			echo '		}'
+			# shellcheck disable=SC2016
+			echo '		$1 ~ /Currently banned/ {'
+			# shellcheck disable=SC2016
+			echo '			print "fail2ban_banned_current{jail=\"'\''"$jail"'\''\"} " $2;'
+			echo '		}'
+			# shellcheck disable=SC2016
+			echo '		$1 ~ /Total banned/ {'
+			# shellcheck disable=SC2016
+			echo '			print "fail2ban_banned_total{jail=\"'\''"$jail"'\''\"} " $2;'
+			echo '		}'\'' '
+			echo '  done | sort | awk '\'' '
+			echo ''
+			echo 'BEGIN {failc=1; failt=1; banc=1; bant=1}'
+			echo '/^fail2ban_failed_current/ { if (failc) {'
+			echo '	print "# HELP fail2ban_failed_current Current number of failures.";'
+			echo '	print "# TYPE fail2ban_failed_current gauge";'
+			echo '	failc=0;'
+			echo '	} }'
+			echo '/^fail2ban_failed_total/ { if (failt) {'
+			echo '	print "# HELP fail2ban_failed_total Total number of failures.";'
+			echo '	print "# TYPE fail2ban_failed_total counter";'
+			echo '	failt=0;'
+			echo '	} }'
+			echo '/^fail2ban_banned_current/ { if (banc) {'
+			echo '	print "# HELP fail2ban_banned_current Current number banned.";'
+			echo '	print "# TYPE fail2ban_banned_current gauge";'
+			echo '	banc=0;'
+			echo '	} }'
+			echo '/^fail2ban_banned_total/ { if (bant) {'
+			echo '	print "# HELP fail2ban_banned_total Total number banned.";'
+			echo '	print "# TYPE fail2ban_banned_total counter";'
+			echo '	bant=0;'
+			echo '	} }'
+			# shellcheck disable=SC2016
+			echo '{ print $0 }'\'' '
+			echo ''
+			echo 'cat <<EOF'
+			echo '# HELP fail2ban_jails Total number of jails.'
+			echo '# TYPE fail2ban_jails gauge'
+			# shellcheck disable=SC2016
+			echo 'fail2ban_jails $jails'
+			echo 'EOF'
+			}  > /usr/local/bin/fail2ban_metrics.sh
+
+		chmod 755 /usr/local/bin/fail2ban_metrics.sh
+	}
+}
+
+install_fail2_metrics

monitor_install.sh

@@ -0,0 +1,743 @@
+#! /bin/bash
+
+#############################################################
+#### Prometheus Monitor install Script for Oracle Linux, #### 
+#### Centos/Redhat and Debian/Ubuntu Servers.            ####       
+####                                                     ####
+#### Author: Phil Connor 08/27/2023                      ####
+#### Contact: pconnor@ara.com                            ####
+#### Version 1.65.031524                                 ####
+####                                                     ####
+#### To use this script chmod it to 755                  ####
+#### or simply type bash <filename.sh>                   ####  
+#############################################################
+
+########################
+### System Variables ###
+########################
+if [ "$(command -v lsb_release)" ]; then
+    OS=$(lsb_release -i | awk '{print $3}' | tr '[:upper:]' '[:lower:]')
+    OSVER=$(lsb_release -r | awk '{print $2}' | awk -F. '{print $1}')
+else 
+    OS=$(grep PRETTY_NAME /etc/os-release | sed 's/PRETTY_NAME=//g' | tr -d '="' | awk '{print $1}' | tr '[:upper:]' '[:lower:]')
+    OSVER=$(grep VERSION_ID /etc/os-release | sed 's/VERSION_ID=//g' | tr -d '="' | awk -F. '{print $1}')
+fi
+
+###########################################################
+#### Detect Package Manger from OS and OSVer Variables ####
+###########################################################
+if [[ "${OS}" = almalinux || "${OS}" = centos || "${OS}" = red || "${OS}" = rocky ]]; then
+    pkgmgr="dnf -y"
+elif [ "${OS}" = ubuntu ]; then
+    pkgmgr="apt -y"
+fi
+
+#####################################
+#### Check for Service Directory ####
+#####################################
+if ! [ -d "/usr/lib/systemd/system/" ]; then
+    psdir='/etc/systemd/system/'
+else
+    psdir='/usr/lib/systemd/system/'
+fi
+
+###########################
+#### Create User/Group ####
+###########################
+if ! grep prometheus /etc/passwd; then
+    groupadd --system prometheus
+    useradd -s /sbin/nologin --system -g prometheus prometheus
+fi
+
+#############################
+#### Check for Directory ####
+#############################
+if [ ! -d "/var/lib/node_exporter/" ]; then
+    mkdir -p /var/lib/node_exporter
+    chown prometheus. /var/lib/node_exporter
+fi
+
+
+###############################
+#### Install Node Exporter ####
+###############################
+install_node_exporter() {
+    {
+        rm -rf /tmp/node_exporter*
+        cd /tmp || exit 2
+        curl -s https://api.github.com/repos/prometheus/node_exporter/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi -
+        tar -xvf node_exporter*.tar.gz
+        cd node_exporter*/ || exit 2
+        mv node_exporter /usr/local/bin
+        chown prometheus. /usr/local/bin/node_exporter
+        
+        if [ "$OS" = red ]; then
+            if [ "$OSVER" = 8 ]; then
+                restorecon -rv /usr/local/bin/node_exporter
+            fi
+        fi
+
+        touch $psdir/node_exporter.service
+        {
+            echo '[Unit]'
+            echo 'Description=Prometheus Node Exporter'
+            echo 'Wants=network-online.target'
+            echo 'After=network-online.target'
+            echo ''
+            echo '[Service]'
+            if [ "$OS" = red ]; then
+                if [ "$OSVER" = 8 ]; then
+                    echo 'User=root'
+                    echo 'Group=root'
+                fi
+            else
+                echo 'User=prometheus'
+                echo 'Group=prometheus'
+            fi
+            echo 'Type=simple'
+            echo "ExecStart=/usr/local/bin/node_exporter \\"
+            echo "    --no-collector.btrfs \\"
+            echo "    --no-collector.hwmon \\"
+            echo "    --no-collector.infiniband \\"
+            echo "    --no-collector.powersupplyclass \\"
+            echo "    --no-collector.thermal_zone \\"
+            echo "    --no-collector.zfs \\"
+            echo "    --collector.ethtool \\"
+            echo "    --collector.interrupts \\"
+            echo "    --collector.processes \\"
+            echo "    --collector.systemd \\"
+            echo "    --collector.tcpstat \\"
+            echo "    --collector.textfile.directory=/var/lib/node_exporter \\"
+            echo '    --collector.filesystem.mount-points-exclude=^/(boot/.+|dev|proc|run|sys|run/.+|var/lib/docker/.+|var/lib/kubelet/.+|var/lib/snap/.+|volumes/user)($|/)'
+            echo ''
+            echo '[Install]'
+            echo 'WantedBy=multi-user.target'
+        } > $psdir/node_exporter.service
+
+        systemctl daemon-reload
+        systemctl enable --now node_exporter
+        #systemctl restart node_exporter
+    }
+}
+
+##################################
+#### Install Loki Log Manager ####
+##################################
+install_loki () {
+    {
+        mkdir -p /etc/loki
+        cd /tmp || exit 2
+
+        wget https://raw.githubusercontent.com/grafana/loki/master/cmd/loki/loki-local-config.yaml
+        wget https://raw.githubusercontent.com/grafana/loki/main/clients/cmd/promtail/promtail-local-config.yaml
+
+        mv /tmp/loki-local-config.yaml /etc/prometheus/loki.yaml
+        mv /tmp/promtail-local-config.yaml /etc/prometheus/promtail.yaml
+
+        {
+            echo 'server:' 
+            echo '  http_listen_port: 9080' 
+            echo '  grpc_listen_port: 0' 
+            echo '' 
+            echo 'positions:' 
+            echo '  filename: /tmp/positions.yaml' 
+            echo '' 
+            echo 'clients:' 
+            echo '  - url: http://localhost:3100/loki/api/v1/push' 
+            echo ''
+            echo 'scrape_configs:' 
+            echo '- job_name: system' 
+            echo '  static_configs:' 
+            echo '  - targets:' 
+            echo '      - localhost' 
+            echo '    labels:' 
+            echo '      job: varlogs' 
+            echo '      __path__: /var/log/*.log # modify to point at your application log file'
+            echo '- job_name: access_log'
+            echo '  static_configs:'
+            echo '  - targets:'
+            echo '      - localhost'
+            echo '  - labels:'
+            echo '      job: apache_logs'
+            echo '      __path__: /var/log/http/*log'
+            echo '  pipeline_stages:'
+            echo '    - regex:'
+            echo '        expression: >-'
+            echo '          ^(?P<ip>.*?) (?P<remote_log_name>.*?)'
+            echo '          (?P<userid>.*?) \[(?P<date>.*?) (?P<timezone>.*?)\]'
+            echo '          \"(?P<request_method>.*?) (?P<path>.*?)'
+            echo '          (?P<request_version>HTTP/.*)?\" (?P<status>.*?)'
+            echo '          (?P<length>.*?) \"(?P<referrer>.*?)\"'
+            echo '          \"(?P<user_agent>.*?)\" (?P<session_id>.*?)'
+            echo '          response-time=(?P<response_time>.*)'
+            echo '    - labels:'
+            echo '        ip:'
+            echo '        userid:'
+            echo '        date:'
+            echo '        timezone:'
+            echo '        request_method:'
+            echo '        path:'
+            echo '        request_version:'
+            echo '        status:'
+            echo '        length:'
+        } > /etc/loki/promtail.yml
+    
+        {
+            echo 'auth_enabled: false' 
+            echo ''
+            echo 'server:' 
+            echo '  http_listen_port: 3100' 
+            echo '  grpc_listen_port: 9096' 
+            echo ''
+            echo 'ingester:' 
+            echo '  wal:'
+            echo '    enabled: true' 
+            echo '    dir: /tmp/wal' 
+            echo '  lifecycler:' 
+            echo '    address: 127.0.0.1' 
+            echo '    ring:' 
+            echo '      kvstore:' 
+            echo '        store: inmemory'
+            echo '      replication_factor: 1' 
+            echo '    final_sleep: 0s' 
+            echo '  chunk_idle_period: 1h       # Any chunk not receiving new logs in this time will be flushed' 
+            echo '  max_chunk_age: 1h           # All chunks will be flushed when they hit this age, default is 1h'
+            echo '  chunk_target_size: 1048576  # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first'
+            echo '  chunk_retain_period: 30s    # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)'
+            echo '  max_transfer_retries: 0     # Chunk transfers disabled'
+            echo '' 
+            echo 'schema_config:' 
+            echo '  configs:' 
+            echo '    - from: 2020-10-24' 
+            echo '      store: boltdb-shipper' 
+            echo '      object_store: filesystem' 
+            echo '      schema: v11'
+            echo '      index:' 
+            echo '        prefix: index'_ 
+            echo '        period: 24h' 
+            echo ''
+            echo 'storage_config:' 
+            echo '  boltdb_shipper:' 
+            echo '    active_index_directory: /tmp/loki/boltdb-shipper-active' 
+            echo '    cache_location: /tmp/loki/boltdb-shipper-cache' 
+            echo '    cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space'
+            echo '    shared_store: filesystem' 
+            echo '  filesystem:' 
+            echo '    directory: /tmp/loki/chunks' 
+            echo '' 
+            echo 'compactor:' 
+            echo '  working_directory: /tmp/loki/boltdb-shipper-compactor' 
+            echo '  shared_store: filesystem' 
+            echo '' 
+            echo 'limits_config:' 
+            echo '  reject_old_samples: true' 
+            echo '  reject_old_samples_max_age: 168h' 
+            echo '' 
+            echo 'chunk_store_config:' 
+            echo '  max_look_back_period: 0s' 
+            echo '' 
+            echo 'table_manager:' 
+            echo '  retention_deletes_enabled: false' 
+            echo '  retention_period: 0s' 
+            echo '' 
+            echo 'ruler:' 
+            echo '  storage:' 
+            echo '    type: local' 
+            echo '    local:' 
+            echo '      directory: /tmp/loki/rules' 
+            echo '  rule_path: /tmp/loki/rules-temp' 
+            echo '  alertmanager_url: http://localhost:9093' 
+            echo '  ring:' 
+            echo '    kvstore:'
+            echo '      store: inmemory' 
+            echo '  enable_api: true'
+        } > /etc/loki/loki.yml
+
+        wget https://github.com/grafana/loki/releases/download/v2.2.1/loki-linux-amd64.zip
+        unzip loki-linux-amd64.zip
+        mv loki-linux-amd64 /usr/local/bin/loki
+        wget https://github.com/grafana/loki/releases/download/v2.2.1/promtail-linux-amd64.zip
+        unzip promtail-linux-amd64.zip
+        mv promtail-linux-amd64 /usr/local/bin/promtail
+
+        {
+            echo '[Unit]' 
+            echo 'Description=Promtail service' 
+            echo 'After=network.target' 
+            echo '' 
+            echo '[Service]' 
+            echo 'Type=simple' 
+            echo 'User=root'
+            echo 'Group=root' 
+            echo "ExecStart=/usr/local/bin/promtail \\"
+            echo '                -config.file /etc/prometheus/promtail.yml' 
+            echo 'Restart=always' 
+            echo '' 
+            echo '[Install]' 
+            echo 'WantedBy=multi-user.target'
+        } > $psdir/promtail.service
+
+        systemctl start promtail.service
+        systemctl enable promtail.service
+
+        {
+            echo '[Unit]' 
+            echo 'Description=Loki service' 
+            echo 'After=network.target' 
+            echo '' 
+            echo '[Service]' 
+            echo 'Type=simple' 
+            echo 'User=prometheus'
+            echo 'Group=prometheus' 
+            echo "ExecStart=/usr/local/bin/loki \\"
+            echo '                -config.file /etc/prometheus/loki.yml'
+            echo 'Restart=always'
+            echo '' 
+            echo '[Install]'
+            echo 'WantedBy=multi-user.target'
+        } > $psdir/loki.service
+
+        systemctl start loki.service
+        systemctl enable loki.service        
+    }
+}
+
+#############################################
+#### Install Audit Directory Size Detect ####
+#############################################
+install_directory_size() {
+    {
+        ############################
+        #### Check for Cron Job ####
+        ############################
+        if ! crontab -l | grep -q 'directory_size.sh'; then
+            echo -e "$(crontab -u root -l)\n*/5 * * * * /usr/local/bin/directory_size.sh /var/log/audit > /var/lib/node_exporter/directory_size.prom 2>&1" | crontab -u root -
+        fi
+
+        #####################
+        ### Create Script ###
+        #####################
+        {
+            # shellcheck disable=SC2028
+            echo '#! /bin/bash'
+            echo ''
+            echo '########################################################################################################'
+            echo '###                                                                                                  ###'
+            echo '### Expose directory usage metrics, passed as an argument.                                           ###'
+            echo '###                                                                                                  ###'
+            echo '### Usage: add this to crontab:                                                                      ###'
+            echo '###                                                                                                  ###'
+            echo '### */5 * * * * directory_size.sh <directory_name> > /var/lib/node_exporter/directory_size.prom 2>&1 ###'
+            echo '###                                                                                                  ###'
+            echo '### Phil Connor pconnor@ara.com                                                                      ###'
+            echo '### Version 1.0.3-111623                                                                             ###'
+            echo '###                                                                                                  ###'
+            echo '########################################################################################################'
+            echo ''
+            echo '###################################'
+            echo '#### Directory Metrics Command ####'
+            echo '###################################'
+            echo 'echo "# HELP node_directory_size_bytes Disk space used by some directories"'
+            echo 'echo "# TYPE node_directory_size_bytes gauge"'
+            # shellcheck disable=SC2028
+            echo 'du --block-size=1 --summarize "$@" | sed -ne '\''s/\\/\\\\/;s/"/\\"/g;s/^\([0-9]\+\)\t\(.*\)$/node_directory_size_bytes{directory="\2"} \1/p'\'' '
+        } > /usr/local/bin/directory_size.sh
+        
+        chmod 755 /usr/local/bin/directory_size.sh
+    } 
+}
+
+#####################################################
+#### Install Yum/Apt Update/Reboot Needed Detect ####
+#####################################################
+install_apt_yum_detect() {
+    {
+        ############################
+        #### Check for Cron Job ####
+        ############################
+        if [ "$OS" = ubuntu ]; then
+            pkg_file=apt_detect
+        else
+            pkg_file=yum_detect
+        fi
+
+        if ! crontab -l | grep -q "$pkg_file.sh"; then
+            echo -e "$(crontab -u root -l)\n*/5 * * * * /usr/local/bin/$pkg_file.sh > /var/lib/node_exporter/$pkg_file.prom 2>&1" | crontab -u root -
+        fi
+
+        #####################
+        ### Create Script ###
+        #####################
+        if [ "$OS" = ubuntu ]; then 
+            touch /usr/local/bin/apt_detect.sh
+            
+            {
+                echo '#!/bin/bash'
+                echo ''
+                echo '#####################################################'
+                echo '###                                               ###'
+                echo '### Description: Expose metrics from apt updates. ###'
+                echo '###                                               ###'
+                echo '### Phil Connor pconnor@ara.com                   ###'
+                echo '### Version 1.2.2.031524                          ###'
+                echo '###                                               ###' 
+                echo '#####################################################'
+                # shellcheck disable=SC1003,SC2016
+                echo 'upgrades="$(/usr/bin/apt-get --just-print upgrade \'
+                # shellcheck disable=SC1003
+                echo '  | /usr/bin/awk -F'\''[()]'\'' \'
+                # shellcheck disable=SC2016
+                echo '      '\''/^Inst/ { sub("^[^ ]+ ", "", $2); gsub(" ","",$2);'
+                # shellcheck disable=SC1003,SC2016,SC2028
+                echo '                 sub("\\[", " ", $2); sub("\\]", "", $2); print $2 }'\'' \'
+                # shellcheck disable=SC1003
+                echo '  | /usr/bin/sort \'
+                # shellcheck disable=SC1003
+                echo '  | /usr/bin/uniq -c \'
+                # shellcheck disable=SC2016,SC2028
+                echo '  | awk '\''{ gsub(/\\\\/, "\\\\", $2); gsub(/\\"/, "\\\"", $2);'
+                # shellcheck disable=SC2016
+                echo '           gsub(/\[/, "", $3); gsub(/\]/, "", $3);'
+                # shellcheck disable=SC2016
+                echo '           print "apt_upgrades_pending{origin=\"" $2 "\",arch=\"" $3 "\"} " $1}'\'' '
+                echo ')"'
+                echo ''
+                # shellcheck disable=SC2016
+                echo 'upgradelist=$(/usr/bin/apt-get --just-print upgrade | grep Inst | awk '\''{gsub(/\(|\)/,"",$4)}1 {gsub(/:/,".",$4)}1 \ ' 
+                # shellcheck disable=SC2016
+                echo '{gsub(/\[|\]/,"",$3)}1 {gsub(/:/," ",$5)}1 {print "apt_upgradelist{pkgname=\"" $2 "\",uvers=\"" $4 "\", overs=\"" $3 "\", repo=\"" $5 "\"} " 1}'\'')'
+                echo ''
+                echo 'echo '\''# HELP apt_upgrades_pending Apt package pending updates by origin.'\'' '
+                echo 'echo '\''# TYPE apt_upgrades_pending gauge'\'' '
+                # shellcheck disable=SC2016
+                echo 'if [[ -n "${upgrades}" ]] ; then'
+                # shellcheck disable=SC2016
+                echo '  echo "${upgrades}"'
+                echo 'else'
+                echo '  echo '\''apt_upgrades_pending{origin="",arch=""} 0'\'' '
+                echo 'fi'
+                echo ''
+                echo 'echo '\''# HELP apt_upgradelist List of packages for upgrade'\'' '
+                echo 'echo '\''# TYPE apt_upgradelist gauge'\'' '
+                # shellcheck disable=SC2016
+                echo 'if [[ -n "${upgradelist}" ]]; then'
+                # shellcheck disable=SC2016
+                echo '    echo "${upgradelist}"'
+                echo 'else'
+                echo '    echo '\''apt_upgradelist{origin="",arch=""} 0'\'' '
+                echo 'fi'
+                echo 'echo '\''# HELP node_reboot_required Node reboot is required for software updates.'\'' '
+                echo '# TYPE node_reboot_required gauge'
+                echo 'if [[ -f '/run/reboot-required' ]] ; then'
+                echo '  echo '\''node_reboot_required 1'\'' '
+                echo 'else'
+                echo '  echo '\''node_reboot_required 0'\'' '
+                echo 'fi'
+            } > /usr/local/bin/apt_detect.sh
+
+            chmod 755 /usr/local/bin/apt_detect.sh
+        else
+            touch /usr/local/bin/yum_detect.sh
+
+            {
+                echo '#! /bin/bash'
+                echo ''
+                echo '#####################################################'
+                echo '###                                               ###'
+                echo '### Description: Expose metrics from yum updates. ###'
+                echo '###                                               ###'
+                echo '### Phil Connor pconnor@ara.com                   ###'
+                echo '### Version 1.1.5.031524                          ###'
+                echo '###                                               ###' 
+                echo '#####################################################'
+                echo 'set -u -o pipefail'
+                echo ''
+                echo '# shellcheck disable=SC2016'
+                echo 'filter_awk_script='\'' '
+                echo 'BEGIN { mute=1 }'
+                echo '/Obsoleting Packages/ {'
+                echo '    mute=0'
+                echo '}'
+                echo 'mute && /^[[:print:]]+\.[[:print:]]+/ {'
+                # shellcheck disable=SC2016
+                echo '    print $3'
+                echo '}'
+                echo ''\'' '
+                echo ''
+                echo 'check_upgrades() {'
+                echo '    {'
+                echo '        /usr/bin/yum -q check-update |'
+                echo '        /usr/bin/xargs -n3 |'
+                # shellcheck disable=SC2016
+                echo '        awk "${filter_awk_script}" |'
+                echo '        sort |'
+                echo '        uniq -c |'
+                # shellcheck disable=SC2016
+                echo '        awk '\''{print "yum_upgrades_pending{origin=\""$2"\"} "$1}'\'' '
+                echo '    }'
+                echo '}'
+                echo ''
+                # shellcheck disable=SC2016
+                echo 'upgrades=$(check_upgrades)'
+                echo ''
+                echo 'echo '\''# HELP yum_upgrades_pending Yum package pending updates by origin.'\'' '
+                echo 'echo '\''# TYPE yum_upgrades_pending gauge'\'' '
+                # shellcheck disable=SC2016
+                echo 'if [[ -n "${upgrades}" ]]; then'
+                # shellcheck disable=SC2016
+                echo '    echo "${upgrades}"'
+                echo 'else'
+                echo '    echo '\''yum_upgrades_pending{origin=""} 0'\'' '
+                echo 'fi'
+                echo ''
+                # shellcheck disable=SC2016
+                echo 'upgradelist=$(yum check-update | awk '\''NR>4 {print "yum_upgradelist{pkgname=\"" $1 "\", uvers=\"" $2 "\", repo=\"" $3 "\"}", 1}'\'')'
+                echo ''
+                echo 'echo '\''# HELP yum_upgradelist List of packages for upgrade'\'' '
+                echo 'echo '\''# TYPE yum_upgradelist gauge'\'' '
+                # shellcheck disable=SC2016
+                echo 'if [[ -n "${upgradelist}" ]]; then'
+                # shellcheck disable=SC2016
+                echo '    echo "${upgradelist}"'
+                echo 'else'
+                echo '    echo '\''yum_upgradelist{pkgname=""} 0'\'' '
+                echo 'fi'
+                echo ''
+                echo '### If yum-utils/dnf-utils is not installed this will skip reboot required metric'
+                echo 'if [[ -x /bin/needs-restarting ]]; then'
+                echo '    echo '\''# HELP node_reboot_required Node reboot is required for software updates.'\'' '
+                echo '    echo '\''# TYPE node_reboot_required gauge'\'' '
+                echo '    if /bin/needs-restarting -r  > /dev/null 2>&1; then'
+                echo '        echo '\''node_reboot_required 0'\'' '
+                echo '    else'
+                echo '        echo '\''node_reboot_required 1'\'' '
+                echo '    fi'
+                echo 'fi'
+            } > /usr/local/bin/yum_detect.sh
+            
+            chmod 755 /usr/local/bin/yum_detect.sh
+        fi
+    }
+}
+
+install_salt_status() {
+    {
+        if [ "$(command -v /bin/ss)" ]; then
+            ssdir=/bin
+        elif [ "$(command -v /usr/bin/ss)" ]; then
+            ssdir=/usr/bin
+        else
+            ssdir=/usr/sbin
+        fi
+
+        ############################
+        #### Check for Cron Job ####
+        ############################
+        if ! crontab -l | grep -q "salt_status.sh"; then
+            echo -e "$(crontab -u root -l)\n*/5 * * * * /usr/local/bin/salt_status.sh > /var/lib/node_exporter/salt_status.prom 2>&1" | crontab -u root -
+        fi
+
+        #####################
+        ### Create Script ###
+        #####################
+        touch /usr/local/bin/salt_status.sh
+
+        {
+            echo '#! /bin/bash'
+            echo '#####################################################'
+            echo '###                                               ###'
+            echo '### Description: Expose metrics from salt-minion. ###'
+            echo '###                                               ###'
+            echo '### Phil Connor pconnor@ara.com                   ###'
+            echo '### Version 1.3.3.122023                          ###'
+            echo '###                                               ###'
+            echo '#####################################################'
+            echo ''
+            echo '## Local Command Variables'
+            # shellcheck disable=SC2016
+            echo 'if [ ! "$(command -v salt-call)" ]; then'
+            echo '    saltck=none'
+            echo 'else'
+            # shellcheck disable=SC2016,SC2028
+            echo '    saltck=$(salt-call test.ping | grep '\''\bTrue\b'\'')'
+            echo 'fi'
+            echo '' 
+            # shellcheck disable=SC2016,SC2028
+            echo "status=\$($ssdir/ss -nt | grep '\b4505\b')"
+            echo ''
+            echo '## Check If minion is connected to port 4505'
+            echo 'echo '\''# HELP minion_connection_status Shows if Salt-Minion is connected to Salt-Master.'\'' '
+            echo 'echo '\''# TYPE minion_connection_status gauge'\'' '
+            # shellcheck disable=SC2016
+            echo 'if [[ -n "${status}" ]]; then'
+            echo '    echo '\''minion_connection_status{status=""} 1'\'' '
+            echo 'else'
+            echo '    echo '\''minion_connection_status{status=""} 0'\'' '
+            echo 'fi'
+            echo ''
+            echo '## Check to see in minion can ping master' 
+            echo 'echo '\''# HELP minion_ping_status Shows if Salt-Minion is able to ping Salt-Master.'\'' '
+            echo 'echo '\''# TYPE minion_ping_status gauge'\'' '
+            # shellcheck disable=SC2016
+            echo 'if [ $saltck = "error" ]; then'
+            echo '    echo '\''minion_ping_status{status=""} 3'\'' '
+            # shellcheck disable=SC2016
+            echo 'elif [ $saltck = none ]; then'
+            echo '    echo '\''minion_ping_status{status=""} 2'\'' '
+            # shellcheck disable=SC2016
+            echo 'elif [[ -n "${saltck}" ]]; then'
+            echo '    echo '\''minion_ping_status{status=""} 1'\'' '
+            echo 'else'
+            echo '    echo '\''minion_ping_status{status=""} 0'\'' '
+            echo 'fi'
+        } > /usr/local/bin/salt_status.sh       
+
+        chmod 755 /usr/local/bin/salt_status.sh       
+    }
+}
+
+install_process_list() {
+    {
+        ############################
+        #### Check for Cron Job ####
+        ############################
+        if ! crontab -l | grep -q "process_list.sh"; then
+            echo -e "$(crontab -u root -l)\n*/3 * * * * /usr/local/bin/process_list.sh > /var/lib/node_exporter/process_list.prom 2>&1" | crontab -u root -
+        fi
+
+        ##################################
+        ### Check if lsof is installed ###
+        ##################################
+        if [ ! "$(command -v lsof)" ]; then
+            $pkgmgr install lsof
+        fi
+
+        ##########################
+        ### Set lsof Dirictory ###
+        ##########################
+        if [ "$(command -v /bin/lsof)" ]; then
+            lfdir=/bin
+        elif [ "$(command -v /usr/bin/lsof)" ]; then
+            lfdir=/usr/bin
+        else
+            lfdir=/usr/sbin
+        fi
+
+        #####################
+        ### Create Script ###
+        #####################
+        touch /usr/local/bin/process_list.sh
+
+        {
+            echo '#! /bin/bash'
+            echo ''
+            echo '########################################################'
+            echo '###                                                  ###'
+            echo '### Description: Expose metrics from cpu by process. ###'
+            echo '###                                                  ###'
+            echo '### Phil Connor pconnor@ara.com                      ###'
+            echo '### Version 2.8.1.031424                             ###'
+            echo '###                                                  ###'
+            echo '########################################################'
+            echo ''
+            echo '#############################'
+            echo '### Process List Function ###'
+            echo '#############################'
+            echo 'processes_list() {'
+            echo '    {'
+            # shellcheck disable=SC2016
+            echo '        PList=$(ps aux)'
+            echo ''
+            echo '        while read -r PList'
+            echo '        do'
+            # shellcheck disable=SC2016
+            echo '            pl=$(/usr/bin/awk '\''{print "node_cpu_usage{process=\""$11"\", pid=\""$2"\", owner=\""$1"\"}", $3}'\'')'
+            # shellcheck disable=SC2016
+            echo '        done <<< "$PList"'
+            echo ''
+            echo '        echo '\''# HELP node_cpu_usage Usage of CPU by process.'\'' '
+            echo '        echo '\''# TYPE node_cpu_usage gauge'\'' ' 
+            # shellcheck disable=SC2016
+            echo '        echo "$pl"'
+            echo '    }'
+            echo '}'
+            echo ''
+            echo '##################################'
+            echo '### File Handler List Function ###'
+            echo '##################################'
+            echo 'filehandlers_list() {'
+            echo '    {'
+            # shellcheck disable=SC2016,SC2028
+            echo "        Fhlist=\$($lfdir/lsof | /usr/bin/awk '{gsub(/\\\\/,\"\",\$1)}1 {gsub(/\\\\|\:/,\"\",\$4)}1 {print \$1 \" \" \$2 \" \" \$4}' | sort | uniq -c | sort -rn | head -20)" 
+            echo ''        
+            echo '        while read -r Fhlist'
+            echo '        do'
+            # shellcheck disable=SC2016
+            echo '            fhl=$(/usr/bin/awk '\''{print "node_file_handlers{pid=\""$1"\", program=\""$2"\"}", $3}'\'')'
+            # shellcheck disable=SC2016
+            echo '        done <<< "$Fhlist"'
+            echo '        echo '\''# HELP node_file_handler Usage of File Handlers.'\'' '
+            echo '        echo '\''# TYPE node_file_handler gauge'\'' '
+            # shellcheck disable=SC2016
+            echo '        echo "$fhl"'
+            echo ''        
+            # shellcheck disable=SC2016
+            echo '        fhmax=$(cat /proc/sys/fs/file-nr | awk '\''{print $3}'\'')'
+            # shellcheck disable=SC2016
+            echo '        fhfree=$(cat /proc/sys/fs/file-nr | awk '\''{print $2}'\'')'
+            # shellcheck disable=SC2016
+            echo '        fhopen=$(cat /proc/sys/fs/file-nr | awk '\''{print $1}'\'')'
+            echo '        echo '\''# HELP node_max_files Max File Limit Handlers.'\'' '
+            echo '        echo '\''# TYPE node_max_files gauge'\'' '
+            # shellcheck disable=SC2016
+            echo '        echo "node_total_max_files_handles $fhmax"'
+            # shellcheck disable=SC2016
+            echo '        echo "node_total_free_file_handles $fhfree"'
+            # shellcheck disable=SC2016
+            echo '        echo "node_total_open_file_handles $fhopen"'
+            echo ''
+            echo '    }'
+            echo '}'
+            echo ''
+            echo '################################'
+            echo '### File Handler Connections ###'
+            echo '################################'    
+            echo 'filehandler_connections() {'
+            echo '    {'
+            # shellcheck disable=SC2016,SC2028
+            echo "        fhconn=\$($lfdir/lsof -i | /usr/bin/awk '{gsub(/\\\\/,\"_\",\$1)}1 {gsub(/\\*|\\:/\"_\",\$2)}1 {gsub(/\\*|\\:|\\\\/,\"\",\$9)}1')"
+            echo ''
+            echo '        while read -r fhconn'
+            echo '            do'
+            # shellcheck disable=SC2016
+            echo '                conn=$(/usr/bin/awk '\''NR>1 {print "node_file_handle_connection{command=\""$1"\", connection=\""$9"\", user=\""$3"\", protocol=\""$5"\", type=\""$8"\"}", $2}'\'')'
+            # shellcheck disable=SC2016
+            echo '            done <<< "$fhconn"'
+            echo '        echo '\''# HELP node_file_handle_connection Connections by process.'\'' '
+            echo '        echo '\''# TYPE node_file_handle_connection gauge'\'' '
+            # shellcheck disable=SC2016
+            echo '        echo "$conn"'
+            echo '    }'
+            echo '}'
+            echo ''
+            echo '######################'
+            echo '### Function Calls ###'
+            echo '######################'
+            echo 'processes_list'
+            echo 'filehandlers_list'
+            echo 'filehandler_connections'
+        } > /usr/local/bin/process_list.sh
+
+        chmod 755 /usr/local/bin/process_list.sh
+
+    }
+}
+########################
+#### Function Calls ####
+########################
+install_node_exporter
+#install_loki
+#install_directory_size
+install_apt_yum_detect
+install_salt_status
+install_process_list

processes_usage.sh

@@ -0,0 +1,161 @@
+#! /bin/bash
+
+########################
+### System Variables ###
+########################
+if [ "$(command -v lsb_release)" ]; then
+    OS=$(lsb_release -i | awk '{print $3}' | tr '[:upper:]' '[:lower:]')
+    OSVER=$(lsb_release -r | awk '{print $2}' | awk -F. '{print $1}')
+else 
+    OS=$(grep PRETTY_NAME /etc/os-release | sed 's/PRETTY_NAME=//g' | tr -d '="' | awk '{print $1}' | tr '[:upper:]' '[:lower:]')
+    OSVER=$(grep VERSION_ID /etc/os-release | sed 's/VERSION_ID=//g' | tr -d '="' | awk -F. '{print $1}')
+fi
+
+###########################################################
+#### Detect Package Manger from OS and OSVer Variables ####
+###########################################################
+if [[ "${OS}" = amazon || "${OS}" = red ]]; then
+        if [[ "${OSVER}" = 2 || "${OSVER}" = 7 ]]; then
+                pkgmgr="yum -y"
+        else
+                pkgmgr="dnf -y"
+        fi
+elif [ "${OS}" = ubuntu ]; then
+        pkgmgr="apt -y"
+fi
+
+#############################
+#### Check for Directory ####
+#############################
+if [ ! -d "/var/lib/node_exporter/" ]; then
+    mkdir -p /var/lib/node_exporter
+    chown prometheus. /var/lib/node_exporter
+fi
+
+############################
+#### Check for Cron Job ####
+############################
+if ! crontab -l | grep -q "cpu_usage.sh"; then
+    echo -e "$(crontab -u root -l)\n*/3 * * * * /usr/local/bin/cpu_usage.sh > /var/lib/node_exporter/cpu_usage.prom 2>&1" | crontab -u root -
+fi
+
+if [ ! "$(command -v lsof)" ]; then
+    $pkgmgr install lsof
+fi
+
+touch /usr/local/bin/cpu_usage.sh
+
+{
+    echo '#! /bin/bash'
+    echo ''
+    echo '########################################################'
+    echo '###                                                  ###'
+    echo '### Description: Expose metrics from cpu by process. ###'
+    echo '###                                                  ###'
+    echo '### Phil Connor pconnor@ara.com                      ###'
+    echo '### Version 2.7.8.020524                             ###'
+    echo '###                                                  ###'
+    echo '########################################################'
+    echo ''
+    echo '#############################'
+    echo '### Process List Function ###'
+    echo '#############################'
+    echo 'processes_list() {'
+    echo '    {'
+    # shellcheck disable=SC2016
+    echo '        PList=$(ps aux)'
+    echo ''
+    echo '        while read -r PList'
+    echo '        do'
+    # shellcheck disable=SC2016
+    echo '            pl=$(/usr/bin/awk '\''{print "node_cpu_usage{process=\""$11"\", pid=\""$2"\", owner=\""$1"\"}", $3}'\'')'
+    # shellcheck disable=SC2016
+    echo '        done <<< "$PList"'
+    echo ''
+    echo '        echo '\''# HELP node_cpu_usage Usage of CPU by process.'\'' '
+    echo '        echo '\''# TYPE node_cpu_usage gauge'\'' ' 
+    # shellcheck disable=SC2016
+    echo '        echo "$pl"'
+    echo '    }'
+    echo '}'
+    echo ''
+    echo '##################################'
+    echo '### File Handler List Function ###'
+    echo '##################################'
+    echo 'filehandlers_list() {'
+    echo '    {'
+    # shellcheck disable=SC2016,SC2028
+    echo '        Fhlist=$(/usr/sbin/lsof | /usr/bin/awk '\''{gsub(/\\/,"",$1)}1 {gsub(/\\|\:/,"",$4)}1 {print $1 " " $2 " " $4}'\'' | sort | uniq -c | sort -rn | head -30)' 
+    echo ''        
+    echo '        while read -r Fhlist'
+    echo '        do'
+    # shellcheck disable=SC2016
+    echo '            fhl=$(/usr/bin/awk '\''{print "node_file_handlers{pid=\""$1"\", program=\""$2"\"}", $3}'\'')'
+    # shellcheck disable=SC2016
+    echo '        done <<< "$Fhlist"'
+    echo '        echo '\''# HELP node_file_handler Usage of File Handlers.'\'' '
+    echo '        echo '\''# TYPE node_file_handler gauge'\'' '
+    # shellcheck disable=SC2016
+    echo '        echo "$fhl"'
+    echo ''        
+    # shellcheck disable=SC2016
+    echo '        fhmax=$(cat /proc/sys/fs/file-nr | awk '\''{print $3}'\'')'
+    # shellcheck disable=SC2016
+    echo '        fhfree=$(cat /proc/sys/fs/file-nr | awk '\''{print $2}'\'')'
+    # shellcheck disable=SC2016
+    echo '        fhopen=$(cat /proc/sys/fs/file-nr | awk '\''{print $1}'\'')'
+    echo '        echo '\''# HELP node_max_files Max File Limit Handlers.'\'' '
+    echo '        echo '\''# TYPE node_max_files gauge'\'' '
+    # shellcheck disable=SC2016
+    echo '        echo "node_total_max_files_handles $fhmax"'
+    # shellcheck disable=SC2016
+    echo '        echo "node_total_free_file_handles $fhfree"'
+    # shellcheck disable=SC2016
+    echo '        echo "node_total_open_file_handles $fhopen"'
+    echo ''
+    echo '    }'
+    echo '}'
+    echo ''
+    echo '################################'
+    echo '### File Handler Connections ###'
+    echo '################################'    
+    echo 'filehandler_connections() {'
+    echo '    {'
+    # shellcheck disable=SC2016,SC2028
+    echo '        fhconn=$(/usr/sbin/lsof -i | /usr/bin/awk '\''{gsub(/\\/,"_",$1)}1 {gsub(/\*|\:/"_",$2)}1 {gsub(/\*|\:|\\/,"",$9)}1'\'')'
+    echo ''
+    echo '        while read -r fhconn'
+    echo '            do'
+    # shellcheck disable=SC2016
+    echo '                conn=$(/usr/bin/awk '\''NR>1 {print "node_file_handle_connection{command=\""$1"\", connection=\""$9"\", user=\""$3"\", protocol=\""$5"\", type=\""$8"\"}", $2}'\'')'
+    # shellcheck disable=SC2016
+    echo '            done <<< "$fhconn"'
+    echo '        echo '\''# HELP node_file_handle_connection Connections by process.'\'' '
+    echo '        echo '\''# TYPE node_file_handle_connection gauge'\'' '
+    # shellcheck disable=SC2016
+    echo '        echo "$conn"'
+    echo '    }'
+    echo '}'
+    echo ''
+    echo '######################'
+    echo '### Function Calls ###'
+    echo '######################'
+    echo 'processes_list'
+    echo 'filehandlers_list'
+    echo 'filehandler_connections'
+} > /usr/local/bin/cpu_usage.sh
+
+chmod 755 /usr/local/bin/cpu_usage.sh
+
+
+#################
+### SEARCH="java"
+### for i in $(ps -C "${SEARCH}" -o pid | grep -v PID); do        echo "PID # ${i} open files count : $(sudo ls -l /proc/${i}/fd | wc -l)"; done
+###############
+
+###############
+### fuser -vm / 2>&1 | awk '$3 ~ /f|F/' | while read user pid flags rest; do printf '%10s %10s %10s %s\n' $user $pid $flags "$(</proc/$pid/cmdline)"; done
+##############
+
+
+

prometheus_install.sh

@@ -0,0 +1,671 @@
+#! /bin/bash
+
+#############################################################
+#### Prometheus install Script for Oracle Linux,         #### 
+#### Centos/Redhat and Debian/Ubuntu Servers.            ####       
+####                                                     ####
+#### Author: Phil Connor 08/27/2023                      ####
+#### Contact: pconnor@ara.com                            ####
+#### Version 2.01.100423                                 ####
+####                                                     ####
+#### To use this script chmod it to 755                  ####
+#### or simply type bash <filename.sh>                   ####  
+#############################################################
+
+########################
+#### User Variables ####
+########################
+domain=mylinux.work
+email=phil@$domain
+########################
+### System Variables ###
+########################
+if [ "$(command -v lsb_release)" ]; then
+    OS=$(lsb_release -i | awk '{print $3}' | tr '[:upper:]' '[:lower:]')
+else
+    OS=$(grep PRETTY_NAME /etc/os-release | sed 's/PRETTY_NAME=//g' | tr -d '="' | awk '{print $1}' | tr '[:upper:]' '[:lower:]')
+fi
+
+PYAML=/etc/prometheus
+
+if ! [ -d "/usr/lib/systemd/system" ]; then
+    psdir='/etc/systemd/system'
+else
+    psdir='/usr/lib/systemd/system'
+fi
+
+#########################
+### Check permissions ###
+#########################
+if [[ $EUID -ne 0 ]]; then
+    echo ''
+    echo "This script must be run as root! Login as root, or sudo/su." 
+    echo ''
+    exit 1;
+fi
+
+######################
+### Package Manager ##
+######################
+if [ "$OS" = ubuntu ]; then
+    pkgmgr='apt -y'
+else
+    pkgmgr='dnf -y'
+fi
+
+###################################
+#### Add Prometheus User/Group ####
+###################################
+if ! grep prometheus /etc/passwd; then 
+    groupadd --system prometheus
+    if [ "$OS" = ubuntu ]; then
+        useradd -s /sbin/nologin --system -g prometheus prometheus
+    else
+        useradd -m -s /bin/false prometheus -g prometheus
+    fi
+fi
+
+#################################
+#### Check for wget and curl #### 
+#################################
+if [ ! "$(command -v wget)" ]; then
+    $pkgmgr install wget
+fi
+
+if [ ! "$(command -v curl)" ]; then
+    $pkgmgr install curl
+fi
+
+##########################
+### Install Prometheus ###
+##########################
+install_prometheus() {
+    {
+        mkdir /etc/prometheus
+        mkdir /var/lib/prometheus
+        chown prometheus /var/lib/prometheus/
+        
+        for i in backups rules templates consoles console_libraries
+        do 
+            mkdir -p /etc/prometheus/${i} 
+            chown -R prometheus. /etc/prometheus/${i}
+            chmod -R 755 /etc/prometheus/${i}
+        done
+
+        cd /tmp || exit 2
+        curl -s https://api.github.com/repos/prometheus/prometheus/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi -
+        tar -xvf prometheus*.tar.gz
+        cd prometheus-*/ || exit 2
+        mv prometheus promtool /usr/local/bin/
+        chown prometheus. /usr/local/bin/prometheus /usr/local/bin/promtool
+        mv prometheus.yml /etc/prometheus/
+        mv consoles/ console_libraries/ /etc/prometheus/
+        chown -R prometheus. /var/lib/prometheus/
+
+        if [ "$OS" = red ]; then
+            if [ "$OSVER" = 8 ]; then
+                restorecon -rv /usr/local/bin/node_exporter
+            fi
+        fi
+
+        cp $PYAML/prometeus.yml $PYAML/backups/
+        {
+            echo '# Global config'
+            echo 'global:'
+            echo '  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. '
+            echo '  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. '
+            echo '  scrape_timeout: 15s  # scrape_timeout is set to the global default (10s).'
+            echo ''
+            echo '# Alertmanager configuration'
+            echo 'alerting:'
+            echo '  alertmanagers:'
+            echo '  - static_configs:'
+            echo '    - targets:'
+            echo '      - alertmanager:9093'
+            echo ''
+            echo '# Load rules once and periodically evaluate them according to the global '\''evaluation_interval'\''.'
+            echo 'rule_files:'
+            echo '# - "first_rules.yml"'
+            echo '# - "second_rules.yml"'
+            echo ''
+            echo '# A scrape configuration containing exactly one endpoint to scrape:# Here it'\''s Prometheus itself.'
+            echo 'scrape_configs:'
+            echo '  # The job name is added as a label '\''job=<job_name>'\'' to any timeseries scraped from this config.'
+            echo '  - job_name: '\''prometheus'\'''
+            echo ''
+            echo '    # metrics_path defaults to '/metrics''
+            echo '    # scheme defaults to '\''http'\''.'
+            echo ''
+            echo '    static_configs:'
+            echo '      - targets: ['\''localhost:9090'\'']'
+            echo ''
+            echo '  - job_name: '\''server_metrics'\'''
+            echo '    scrape_interval: 5s'
+            echo '    static_configs:'
+            echo '      - targets: ['\''localhost:9100'\'']'
+            echo '        labels:'
+            echo '          alias: Prometheus Server'
+        } > /etc/prometheus/prometheus.yml
+
+        ### Not required used for my test machine ###
+        # firewall-cmd --add-port=9090/tcp
+        # firewall-cmd --add-port=9090/tcp --permanent
+        # 
+        
+        touch $psdir/prometheus.service
+
+        {
+            echo '[Unit]'
+            echo 'Description=Prometheus Time Series Collection and Processing Server'
+            echo 'Documentation=https://prometheus.io/docs/introduction/overview/'
+            echo 'Wants=network-online.target'
+            echo 'After=network-online.target'
+            echo ''
+            echo '[Service]'
+            echo 'Type=simple'
+            echo 'User=prometheus'
+            echo 'Group=prometheus'
+            echo ''
+            echo "ExecReload=/bin/kill -HUP \$MAINPID"
+            echo "ExecStart=/usr/local/bin/prometheus \\"
+            echo "    --config.file /etc/prometheus/prometheus.yml \\"
+            echo "    --storage.tsdb.path /var/lib/prometheus/data \\"
+            echo "    --web.console.templates=/etc/prometheus/consoles \\"
+            echo "    --web.console.libraries=/etc/prometheus/console_libraries \\"
+            echo "    --web.listen-address=0.0.0.0:9090 \\"
+            echo "    --web.external-url= \\"
+            echo '    --enable-feature=new-service-discovery-manager,exemplar-storage,extra-scrape-metrics'
+            echo ''
+            echo 'Restart=always'
+            echo 'RestartSec=5s'
+            if [ "$OS" = ubuntu ]; then
+                echo 'SyslogIdentifier=prometheus'
+            fi
+            echo 'Restart=always'
+            echo ''
+            echo '[Install]'
+            echo 'WantedBy=multi-user.target'
+        } > $psdir/prometheus.service
+            
+        systemctl daemon-reload
+        systemctl enable --now prometheus
+        
+    }
+}
+
+#############################
+### Install node_exporter ###
+#############################
+install_node_exporter() {
+    {
+        cd /tmp || exit 2
+        curl -s https://api.github.com/repos/prometheus/node_exporter/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi -
+        tar -xvf node_exporter*.tar.gz
+        cd node_exporter-*/ || exit 2
+        mv node_exporter /usr/local/bin
+        chown prometheus. /usr/local/bin/node_exporter
+        
+        if [ "$OS" = red ]; then
+            if [ "$OSVER" = 8 ]; then
+                restorecon -rv /usr/local/bin/node_exporter
+            fi
+        fi
+
+        touch $psdir/node_exporter.service
+        {
+            echo '[Unit]'
+            echo 'Description=Prometheus Node Exporter'
+            echo 'Wants=network-online.target'
+            echo 'After=network-online.target'
+            echo ''
+            echo '[Service]'
+            echo 'User=prometheus'
+            echo 'Group=prometheus'
+            echo 'Type=simple'
+            echo "ExecStart=/usr/local/bin/node_exporter \\"
+            echo "    --collector.ethtool \\"
+            echo "    --collector.interrupts \\"
+            echo "    --collector.processes \\"
+            echo "    --collector.systemd \\"
+            echo '    --collector.tcpstat'
+            echo ''
+            echo '[Install]'
+            echo 'WantedBy=multi-user.target'
+        } > $psdir/node_exporter.service
+
+        systemctl daemon-reload
+        systemctl enable --now node_exporter
+        
+        ### Not required used for my test machine ###
+        # firewall-cmd --add-port=9100/tcp
+        # firewall-cmd --add-port=9100/tcp  --permanent
+
+        systemctl restart prometheus
+    }
+}
+
+########################
+### Install BlackBox ###
+########################
+install_blackbox() {
+    {
+        cd /tmp || exit 2
+        curl -s https://api.github.com/repos/prometheus/blackbox_exporter/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi -
+        tar -xvf blackbox_exporter*.tar.gz
+        cd blackbox_exporter-*/ || exit 2
+        mv blackbox_exporter /usr/local/bin
+        chown prometheus. /usr/local/bin/blackbox_exporter
+        mv  blackbox.yml /etc/prometheus/
+        chown -R prometheus. /etc/prometheus/
+
+        if [ "$OS" = red ]; then
+            if [ "$OSVER" = 8 ]; then
+                restorecon -rv /usr/local/bin/node_exporter
+            fi
+        fi
+
+        touch $psdir/blackbox_exporter.service
+        
+        {
+            echo '[Unit]'
+            echo 'Description=Prometheus Blackbox Exporter Http/Https Montoring'
+            echo 'After=network.target'
+            echo ''
+            echo '[Service]'
+            echo 'User=prometheus'
+            echo 'Group=prometheus'
+            echo 'Type=simple'
+            echo "ExecStart=/usr/local/bin/blackbox_exporter \\"
+            echo "    --config.file /etc/prometheus/blackbox.yml \\"
+            echo '    --web.listen-address=":9115"'
+            echo ''
+            echo 'Restart=always'
+            echo ''
+            echo '[Install]'
+            echo 'WantedBy=multi-user.target'
+        } > $psdir/blackbox_exporter.service
+
+        systemctl daemon-reload
+        systemctl enable --now blackbox_exporter
+
+        {
+            echo '  - job_name: '"'blackbox'"''
+            echo '    metrics_path: /probe'
+            echo '    params:'
+            echo '      module: [http_2xx]' 
+            echo '    static_configs:'
+            echo '      - targets:'
+            echo '      #### Local Targets ####'
+            echo '         - http://localhost:9090'
+            echo ''
+            echo '      #### Remote Targets ####'
+            echo '         #- https://google.com'
+            echo ''
+            echo '    relabel_configs:'
+            echo '     - source_labels: [__address__]'
+            echo '       target_label: __param_target'
+            echo '     - source_labels: [__param_target]'
+            echo '       target_label: instance'
+            echo '     - target_label: __address__'
+            echo '       replacement: localhost:9115'
+        } >> $PYAML/prometheus.yml
+    
+        cp $PYAML/blackbox.yml $PYAML/backups/
+        
+        {
+            echo 'modules:'
+            echo '  http_2xx:'
+            echo '    prober: http'
+            echo '    timeout: 20s'
+            echo '    http:'
+            echo '      valid_status_codes: []'
+            echo '      valid_http_versions: ["HTTP/1.1", "HTTP/2"]'
+            echo '      ip_protocol_fallback: false'
+            echo '      method: GET'
+            echo '      follow_redirects: true'
+            echo '      preferred_ip_protocol: "ip4"'
+            echo '      fail_if_ssl: false'
+            echo '      fail_if_not_ssl: false'
+            echo '      tls_config:'
+            echo '        insecure_skip_verify: true'
+            echo '      #basic_auth:'
+            echo '         #username: "username"'
+            echo '         #password: "password"'
+            echo '  http_post_2xx:'
+            echo '    prober: http'
+            echo '    http:'
+            echo '      method: POST'
+            echo '  tcp_connect:'
+            echo '    prober: tcp'
+            echo '  pop3s_banner:'
+            echo '    prober: tcp'
+            echo '    tcp:'
+            echo '      query_response:'
+            echo '      - expect: "^+OK"'
+            echo '      tls: true'
+            echo '      tls_config:'
+            echo '        insecure_skip_verify: false'
+            echo '  grpc:'
+            echo '    prober: grpc'
+            echo '    grpc:'
+            echo '      tls: true'
+            echo '      preferred_ip_protocol: "ip4"'
+            echo '  grpc_plain:'
+            echo '    prober: grpc'
+            echo '    grpc:'
+            echo '      tls: false'
+            echo '      service: "service1"'
+            echo '  ssh_banner:'
+            echo '    prober: tcp'
+            echo '    tcp:'
+            echo '      query_response:'
+            echo '      - expect: "^SSH-2.0-"'
+            echo '      - send: "SSH-2.0-blackbox-ssh-check"'
+            echo '  irc_banner:'
+            echo '    prober: tcp'
+            echo '    tcp:'
+            echo '      query_response:'
+            echo '      - send: "NICK prober"'
+            echo '      - send: "USER prober prober prober :prober"'
+            echo '      - expect: "PING :([^ ]+)"'
+            echo '        send: "PONG '\$'{1}"'
+            echo '      - expect: "^:[^ ]+ 001"'
+            echo '  icmp:'
+            echo '    prober: icmp'
+            echo '  icmp_ttl5:'
+            echo '    prober: icmp'
+            echo '    timeout: 5s'
+            echo '    icmp:'
+            echo '      ttl: 5'
+        } > $PYAML/blackbox.yml
+        
+        if ! blackbox_exporter --config.check --config.file $PYAML/blackbox.yml; then
+            echo "There's a problem with the blackbox_exporter.yml config file" 
+            exit 2
+        else 
+            systemctl restart blackbox_exporter
+        fi
+
+        ### Not required used for my test machine ###
+        # firewall-cmd --add-port=9115/tcp
+        # firewall-cmd --add-port=9115/tcp --permanent
+    }
+}
+
+############################
+### Install AlertManager ###
+############################
+install_alertmanager() {
+    {
+        cd /tmp || exit 2
+        curl -s https://api.github.com/repos/prometheus/alertmanager/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi -
+        tar -xvf alertmanager*.tar.gz
+        cd alertmanager-*/ || exit 2
+        mv amtool alertmanager /usr/local/bin
+        mv  alertmanager.yml /etc/prometheus
+        chown -R prometheus. /etc/prometheus
+        chown prometheus. /usr/local/bin/alertmanager /usr/local/bin/amtool
+        mkdir /var/lib/alertmanager
+        chown prometheus. /var/lib/alertmanager
+
+        if [ "$OS" = red ]; then
+            if [ "$OSVER" = 8 ]; then
+                restorecon -rv /usr/local/bin/alertmanager
+            fi
+        fi
+
+        touch $psdir/alertmanager.service
+        
+        {
+        echo '[Unit]'
+        echo 'Description=Prometheus AlertManager Service'
+        echo 'Wants=network-online.target'
+        echo 'After=network-online.target'
+        echo ''
+        echo '[Service]'
+        echo 'User=prometheus'
+        echo 'Group=prometheus'
+        echo 'Type=simple'
+        echo "ExecStart=/usr/local/bin/alertmanager \\"
+        echo "    --config.file /etc/prometheus/alertmanager.yml \\"
+        echo "    --storage.path /var/lib/alertmanager/ \\"
+        echo '    --cluster.advertise-address=0.0.0.0:9093'
+        echo ''
+        echo '[Install]'
+        echo 'WantedBy=multi-user.target'
+        } > $psdir/alertmanager.service
+
+        cp $PYAML/alertmanager.yml $PYAML/backups/
+
+        {
+            echo 'global:'
+            echo '  smtp_smarthost: '\''nlb-exch-imaps-smt-x28bvtlo2e4rs-60fe7830ff427ab2.elb.us-gov-west-1.amazonaws.com:25'\'''
+            echo '   smtp_from: '\''alertmanager@calormen.net'\'''
+            echo '   smtp_require_tls: false'
+            echo ''
+            echo 'templates:'
+            echo '  - '\''/etc/prometheus/templates/email_alert.tmpl'\'''
+            echo ''
+            echo 'route:'
+            echo '  receiver: email'
+            echo ''
+            echo 'receivers:'
+            echo '  - name: '\''email'\'''
+            echo '    email_configs:'
+            echo "      - to: '$email'"
+            echo '        from: '\''prometheus@us.calormen.net'\'''
+            echo '        #html: '\''{{ template "email" .}}'\'''
+            echo '        send_resolved: true'
+        } > $PYAML/alertmanager.yml
+
+        touch $PYAML/templates/email_alerts.tmpl
+
+        {
+            echo '{{ define "email" }}'
+            echo ''
+            echo '<html>'
+            echo '   <head>'
+            echo '      <style type="text/css">'
+            echo '         table {'
+            echo '         font-family: verdana,arial,sans-serif;'
+            echo '         font-size:11px;'
+            echo '         color:#333333;'
+            echo '         border-width: 1px;'
+            echo '         border-color: #999999;'
+            echo '         border-collapse: collapse;'
+            echo '         }'
+            edho '         table th {'
+            echo '         background-color:#ff6961;'
+            echo '         border-width: 1px;'
+            echo '         padding: 8px;'
+            echo '         border-style: solid;'
+            echo '         border-color: #F54C44;'
+            echo '         }'
+            echo '         table td {'
+            echo '         border-width: 1px;'
+            echo '         padding: 8px;'
+            echo '         border-style: solid;'
+            echo '         border-color: #F54C44;'
+            echo '         text-align: right;'
+            echo '         }'
+            echo '      </style>'
+            echo '   </head>'
+            echo '   <body>'
+            echo '      <table border=1>'
+            echo '         <thead>'
+            echo '           <tr>'
+            echo '        <th>Alert name</th>'
+            echo '        <th>Host</th>'
+            echo '            <th>Summary</th>'
+            echo '            <th>Description</th>'
+            echo '           </tr>'
+            echo '         </thead>'
+            echo ''
+            echo '         <tbody>'
+            echo '       {{ range .Alerts }}'
+            echo '            <tr>'
+            echo '         <td>{{ .Labels.alertname }}</td>'
+            echo '         <td>{{ .Annotations.host }}</td>'
+            echo '         <td>{{ .Annotations.summary }}</td>'
+            echo '         <td>{{ .Annotations.description }}</td>'
+            echo '           </tr>'
+            echo '          {{ end }}'
+            echo '         </tbody>'
+            echo ''
+            echo '      </table>'
+            echo '  </body>'
+            echo '</html>'
+            echo ''
+            echo '{{end}}'
+        } > $PYAML/templates/email_alerts.tmpl
+
+        systemctl daemon-reload
+        systemctl --now enable alertmanager
+
+    }
+}
+
+#######################
+### Install Grafana ###
+#######################
+install_grafana() {
+    {
+        if [ "$OS" = ubuntu ]; then 
+            $pkgmgr install -y apt-transport-https software-properties-common
+            mkdir -p /etc/apt/keyrings/
+            wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/grafana.gpg > /dev/null
+            echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" | sudo tee -a /etc/apt/sources.list.d/grafana.list
+            $pkgmgr update
+            $pkgmgr install grafana
+        else
+            touch /etc/yum.repos.d/grafana.repo
+            {
+                echo '[grafana]'
+                echo 'name=grafana'
+                echo 'baseurl=https://packages.grafana.com/oss/rpm'
+                echo 'repo_gpgcheck=1'
+                echo 'enabled=1'
+                echo 'gpgcheck=1'
+                echo 'gpgkey=https://packages.grafana.com/gpg.key'
+                echo 'sslverify=1'
+                echo 'sslcacert=/etc/pki/tls/certs/ca-bundle.crt'
+            } > /etc/yum.repos.d/grafana.repo
+
+            dnf -y repolist 
+            dnf -y install grafana
+        fi
+
+        systemctl daemon-reload
+        systemctl enable --now grafana-server
+
+        ### Not required used for my test machine ###
+        # firewall-cmd --add-port=3000/tcp 
+        # firewall-cmd --add-port=3000/tcp --permanent
+    }
+}
+
+install_nginx() {
+    {
+        $pkgmgr install nginx
+        sitesa=/etc/nginx/sites-available
+        sitese=/etc/nginx/sites-enabled/
+
+        touch $sitesa/prometheus.conf
+        {
+            echo 'server {'
+            echo '    listen 80;'
+            echo '    listen [::]:80;'
+            echo ''    
+            echo "    server_name prometheus.$domain;"
+            echo ''    
+            echo '    location / {'
+            echo '        proxy_pass http://localhost:9090/;'
+            echo '        include proxy_params;'
+            echo '        }'
+            echo '}'
+        } > $sitesa/prometheus.conf
+
+        touch $sitesa/metrics.conf
+        {
+            echo 'server {'
+            echo '    listen 80;'
+            echo '    listen [::]:80;'
+            echo ''    
+            echo "    server_name metrics.$domain;"
+            echo ''    
+            echo '    location / {'
+            echo '        proxy_pass http://localhost:3000/;'
+            echo '        include proxy_params;'
+            echo '        }'
+            echo '}'
+        } > $sitesa/metrics.conf
+
+        touch $sitesa/alerts.conf
+        {
+            echo 'server {'
+            echo '    listen 80;'
+            echo '    listen [::]:80;'
+            echo ''    
+            echo "    server_name alerts.$domain;"
+            echo ''    
+            echo '    location / {'
+            echo '        proxy_pass http://localhost:9093/;'
+            echo '        include proxy_params;'
+            echo '        }'
+            echo '}'
+        } > $sitesa/alerts.conf
+
+                touch $sitesa/loki.conf
+        {
+            echo 'server {'
+            echo '    listen 80;'
+            echo '    listen [::]:80;'
+            echo ''    
+            echo "    server_name loki.$domain;"
+            echo ''    
+            echo '    location / {'
+            echo '        proxy_pass http://localhost:9093/;'
+            echo '        include proxy_params;'
+            echo '        }'
+            echo '}'
+        } > $sitesa/alerts.conf
+        
+        ln -s $sitesa/prometheus.conf $sitese
+        ln -s $sitesa/metrics.conf $sitese
+        ln -s $sitesa/alerts.conf $sitese
+        ln -s $sitesa/prometheus.conf $sitese
+        
+        systemctl restart nginx
+    }
+}
+
+install_certbot() {
+    {
+        systemctl stop nginx
+        if [ "$OS" = ubuntu ]; then
+            $pkgmgr update
+            $pkgmgr install python3-certbot-nginx certbot
+            systemctl enable --now  certbot.timer
+        else
+            $pkgmgr install epel-release
+            $pkgmgr install certbot python3-certbot-nginx
+            touch /etc/cron.weekly/certbot
+            chmod +x /etc/cron.weekly/certbot
+                        {
+                echo '#!/bin/sh'
+                echo 'certbot renew'
+            } > /etc/cron.weekly/certbot
+        fi
+    }
+}
+
+######################
+### Function Calls ###
+######################
+install_prometheus
+install_node_exporter
+install_blackbox
+install_alertmanager
+install_grafana 
+install_nginx
+#install_certbot

salt_status.sh

@@ -0,0 +1,73 @@
+#! /bin/bash
+
+if ! crontab -l | grep -q "salt_status.sh"; then
+    echo -e "$(crontab -u root -l)\n*/5 * * * * /usr/local/bin/salt_status.sh > /var/lib/node_exporter/salt_status.prom 2>&1" | crontab -u root -
+fi
+
+if [ "$(command -v /bin/ss)" ]; then
+    dir=/bin
+elif [ "$(command -v /usr/bin/ss)" ]; then
+    dir=/usr/bin
+else
+    dir=/usr/sbin
+fi
+
+install_salt_status() {
+    {
+        touch /usr/local/bin/salt_status.sh
+
+        {
+            echo '#! /bin/bash'
+            echo '#####################################################'
+            echo '###                                               ###'
+            echo '### Description: Expose metrics from salt-minion. ###'
+            echo '###                                               ###'
+            echo '### Phil Connor contact@mylinux.work              ###'
+            echo '### Version 1.3.3.122023                          ###'
+            echo '###                                               ###'
+            echo '#####################################################'
+            echo ''
+            echo '## Local Command Variables'
+            # shellcheck disable=SC2016
+            echo 'if [ ! "$(command -v salt-call)" ]; then'
+            echo '    cert=none'
+            echo 'else'
+            # shellcheck disable=SC2016,SC2028
+            echo '    cert=$(salt-call test.ping | grep '\''\bTrue\b'\'')'
+            echo 'fi'
+            echo '' 
+            # shellcheck disable=SC2016,SC2028
+            echo "status=\$($dir/ss -nt | grep '\b4505\b')"
+            echo ''
+            echo '## Check If minion is connected to port 4505'
+            echo 'echo '\''# HELP minion_connection_status Shows if Salt-Minion is connected to Salt-Master.'\'' '
+            echo 'echo '\''# TYPE minion_connection_status gauge'\'' '
+            # shellcheck disable=SC2016
+            echo 'if [[ -n "${status}" ]]; then'
+            echo '    echo '\''minion_connection_status{status=""} 1'\'' '
+            echo 'else'
+            echo '    echo '\''minion_connection_status{status=""} 0'\'' '
+            echo 'fi'
+            echo ''
+            echo '## Check to see in minion can ping master' 
+            echo 'echo '\''# HELP minion_ping_status Shows if Salt-Minion is able to ping Salt-Master.'\'' '
+            echo 'echo '\''# TYPE minion_ping_status gauge'\'' '
+            # shellcheck disable=SC2016
+            echo 'if [ $cert = none ]; then'
+            echo '    echo '\''minion_ping_status{status=""} 2'\'' '
+            # shellcheck disable=SC2016
+            echo 'elif [[ -n "${cert}" ]]; then'
+            echo '    echo '\''minion_ping_status{status=""} 1'\'' '
+            echo 'else'
+            echo '    echo '\''minion_ping_status{status=""} 0'\'' '
+            echo 'fi'
+        } > /usr/local/bin/salt_status.sh       
+
+        chmod 755 /usr/local/bin/salt_status.sh     
+    }
+}
+
+########################
+#### Function Calls ####
+########################
+install_salt_status

yum_detect.sh

@@ -0,0 +1,52 @@
+#! /bin/bash
+#####################################
+# Description: Expose metrics from yum updates.
+#
+# Phil Connor pconnor@ara.com
+# Version 1.0.4.110523
+# Based on apt_detect.sh 
+######################################
+set -u -o pipefail
+
+# shellcheck disable=SC2016
+filter_awk_script='
+BEGIN { mute=1 }
+/Obsoleting Packages/ {
+    mute=0
+}
+mute && /^[[:print:]]+\.[[:print:]]+/ {
+    print $3
+}
+'
+
+check_upgrades() {
+    {
+        /usr/bin/yum -q check-update |
+        /usr/bin/xargs -n3 |
+        awk "${filter_awk_script}" |
+        sort |
+        uniq -c |
+        awk '{print "yum_upgrades_pending{origin=\""$2"\"} "$1}'
+    }
+}
+
+upgrades=$(check_upgrades)
+
+echo '# HELP yum_upgrades_pending Yum package pending updates by origin.'
+echo '# TYPE yum_upgrades_pending gauge'
+if [[ -n "${upgrades}" ]] ; then
+    echo "${upgrades}"
+else
+    echo 'yum_upgrades_pending{origin=""} 0'
+fi
+
+# If yum-utils/dnf-utils is not installed then we skip rendering this metric
+if [[ -x /bin/needs-restarting ]] ; then
+    echo '# HELP node_reboot_required Node reboot is required for software updates.'
+    echo '# TYPE node_reboot_required gauge'
+    if /bin/needs-restarting -r  > /dev/null 2>&1 ; then
+        echo 'node_reboot_required 0'
+    else
+        echo 'node_reboot_required 1'
+    fi
+fi