Website/prometheus_install.sh
2024-03-20 17:41:57 -05:00

671 lines
23 KiB
Bash

#! /bin/bash
#############################################################
#### Prometheus install Script for Oracle Linux, ####
#### Centos/Redhat and Debian/Ubuntu Servers. ####
#### ####
#### Author: Phil Connor 08/27/2023 ####
#### Contact: pconnor@ara.com ####
#### Version 2.01.100423 ####
#### ####
#### To use this script chmod it to 755 ####
#### or simply type bash <filename.sh> ####
#############################################################
########################
#### User Variables ####
########################
domain=mylinux.work
email=phil@$domain
########################
### System Variables ###
########################
if [ "$(command -v lsb_release)" ]; then
OS=$(lsb_release -i | awk '{print $3}' | tr '[:upper:]' '[:lower:]')
else
OS=$(grep PRETTY_NAME /etc/os-release | sed 's/PRETTY_NAME=//g' | tr -d '="' | awk '{print $1}' | tr '[:upper:]' '[:lower:]')
fi
PYAML=/etc/prometheus
if ! [ -d "/usr/lib/systemd/system" ]; then
psdir='/etc/systemd/system'
else
psdir='/usr/lib/systemd/system'
fi
#########################
### Check permissions ###
#########################
if [[ $EUID -ne 0 ]]; then
echo ''
echo "This script must be run as root! Login as root, or sudo/su."
echo ''
exit 1;
fi
######################
### Package Manager ##
######################
if [ "$OS" = ubuntu ]; then
pkgmgr='apt -y'
else
pkgmgr='dnf -y'
fi
###################################
#### Add Prometheus User/Group ####
###################################
if ! grep prometheus /etc/passwd; then
groupadd --system prometheus
if [ "$OS" = ubuntu ]; then
useradd -s /sbin/nologin --system -g prometheus prometheus
else
useradd -m -s /bin/false prometheus -g prometheus
fi
fi
#################################
#### Check for wget and curl ####
#################################
if [ ! "$(command -v wget)" ]; then
$pkgmgr install wget
fi
if [ ! "$(command -v curl)" ]; then
$pkgmgr install curl
fi
##########################
### Install Prometheus ###
##########################
install_prometheus() {
{
mkdir /etc/prometheus
mkdir /var/lib/prometheus
chown prometheus /var/lib/prometheus/
for i in backups rules templates consoles console_libraries
do
mkdir -p /etc/prometheus/${i}
chown -R prometheus. /etc/prometheus/${i}
chmod -R 755 /etc/prometheus/${i}
done
cd /tmp || exit 2
curl -s https://api.github.com/repos/prometheus/prometheus/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi -
tar -xvf prometheus*.tar.gz
cd prometheus-*/ || exit 2
mv prometheus promtool /usr/local/bin/
chown prometheus. /usr/local/bin/prometheus /usr/local/bin/promtool
mv prometheus.yml /etc/prometheus/
mv consoles/ console_libraries/ /etc/prometheus/
chown -R prometheus. /var/lib/prometheus/
if [ "$OS" = red ]; then
if [ "$OSVER" = 8 ]; then
restorecon -rv /usr/local/bin/node_exporter
fi
fi
cp $PYAML/prometeus.yml $PYAML/backups/
{
echo '# Global config'
echo 'global:'
echo ' scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. '
echo ' evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. '
echo ' scrape_timeout: 15s # scrape_timeout is set to the global default (10s).'
echo ''
echo '# Alertmanager configuration'
echo 'alerting:'
echo ' alertmanagers:'
echo ' - static_configs:'
echo ' - targets:'
echo ' - alertmanager:9093'
echo ''
echo '# Load rules once and periodically evaluate them according to the global '\''evaluation_interval'\''.'
echo 'rule_files:'
echo '# - "first_rules.yml"'
echo '# - "second_rules.yml"'
echo ''
echo '# A scrape configuration containing exactly one endpoint to scrape:# Here it'\''s Prometheus itself.'
echo 'scrape_configs:'
echo ' # The job name is added as a label '\''job=<job_name>'\'' to any timeseries scraped from this config.'
echo ' - job_name: '\''prometheus'\'''
echo ''
echo ' # metrics_path defaults to '/metrics''
echo ' # scheme defaults to '\''http'\''.'
echo ''
echo ' static_configs:'
echo ' - targets: ['\''localhost:9090'\'']'
echo ''
echo ' - job_name: '\''server_metrics'\'''
echo ' scrape_interval: 5s'
echo ' static_configs:'
echo ' - targets: ['\''localhost:9100'\'']'
echo ' labels:'
echo ' alias: Prometheus Server'
} > /etc/prometheus/prometheus.yml
### Not required used for my test machine ###
# firewall-cmd --add-port=9090/tcp
# firewall-cmd --add-port=9090/tcp --permanent
#
touch $psdir/prometheus.service
{
echo '[Unit]'
echo 'Description=Prometheus Time Series Collection and Processing Server'
echo 'Documentation=https://prometheus.io/docs/introduction/overview/'
echo 'Wants=network-online.target'
echo 'After=network-online.target'
echo ''
echo '[Service]'
echo 'Type=simple'
echo 'User=prometheus'
echo 'Group=prometheus'
echo ''
echo "ExecReload=/bin/kill -HUP \$MAINPID"
echo "ExecStart=/usr/local/bin/prometheus \\"
echo " --config.file /etc/prometheus/prometheus.yml \\"
echo " --storage.tsdb.path /var/lib/prometheus/data \\"
echo " --web.console.templates=/etc/prometheus/consoles \\"
echo " --web.console.libraries=/etc/prometheus/console_libraries \\"
echo " --web.listen-address=0.0.0.0:9090 \\"
echo " --web.external-url= \\"
echo ' --enable-feature=new-service-discovery-manager,exemplar-storage,extra-scrape-metrics'
echo ''
echo 'Restart=always'
echo 'RestartSec=5s'
if [ "$OS" = ubuntu ]; then
echo 'SyslogIdentifier=prometheus'
fi
echo 'Restart=always'
echo ''
echo '[Install]'
echo 'WantedBy=multi-user.target'
} > $psdir/prometheus.service
systemctl daemon-reload
systemctl enable --now prometheus
}
}
#############################
### Install node_exporter ###
#############################
install_node_exporter() {
{
cd /tmp || exit 2
curl -s https://api.github.com/repos/prometheus/node_exporter/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi -
tar -xvf node_exporter*.tar.gz
cd node_exporter-*/ || exit 2
mv node_exporter /usr/local/bin
chown prometheus. /usr/local/bin/node_exporter
if [ "$OS" = red ]; then
if [ "$OSVER" = 8 ]; then
restorecon -rv /usr/local/bin/node_exporter
fi
fi
touch $psdir/node_exporter.service
{
echo '[Unit]'
echo 'Description=Prometheus Node Exporter'
echo 'Wants=network-online.target'
echo 'After=network-online.target'
echo ''
echo '[Service]'
echo 'User=prometheus'
echo 'Group=prometheus'
echo 'Type=simple'
echo "ExecStart=/usr/local/bin/node_exporter \\"
echo " --collector.ethtool \\"
echo " --collector.interrupts \\"
echo " --collector.processes \\"
echo " --collector.systemd \\"
echo ' --collector.tcpstat'
echo ''
echo '[Install]'
echo 'WantedBy=multi-user.target'
} > $psdir/node_exporter.service
systemctl daemon-reload
systemctl enable --now node_exporter
### Not required used for my test machine ###
# firewall-cmd --add-port=9100/tcp
# firewall-cmd --add-port=9100/tcp --permanent
systemctl restart prometheus
}
}
########################
### Install BlackBox ###
########################
install_blackbox() {
{
cd /tmp || exit 2
curl -s https://api.github.com/repos/prometheus/blackbox_exporter/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi -
tar -xvf blackbox_exporter*.tar.gz
cd blackbox_exporter-*/ || exit 2
mv blackbox_exporter /usr/local/bin
chown prometheus. /usr/local/bin/blackbox_exporter
mv blackbox.yml /etc/prometheus/
chown -R prometheus. /etc/prometheus/
if [ "$OS" = red ]; then
if [ "$OSVER" = 8 ]; then
restorecon -rv /usr/local/bin/node_exporter
fi
fi
touch $psdir/blackbox_exporter.service
{
echo '[Unit]'
echo 'Description=Prometheus Blackbox Exporter Http/Https Montoring'
echo 'After=network.target'
echo ''
echo '[Service]'
echo 'User=prometheus'
echo 'Group=prometheus'
echo 'Type=simple'
echo "ExecStart=/usr/local/bin/blackbox_exporter \\"
echo " --config.file /etc/prometheus/blackbox.yml \\"
echo ' --web.listen-address=":9115"'
echo ''
echo 'Restart=always'
echo ''
echo '[Install]'
echo 'WantedBy=multi-user.target'
} > $psdir/blackbox_exporter.service
systemctl daemon-reload
systemctl enable --now blackbox_exporter
{
echo ' - job_name: '"'blackbox'"''
echo ' metrics_path: /probe'
echo ' params:'
echo ' module: [http_2xx]'
echo ' static_configs:'
echo ' - targets:'
echo ' #### Local Targets ####'
echo ' - http://localhost:9090'
echo ''
echo ' #### Remote Targets ####'
echo ' #- https://google.com'
echo ''
echo ' relabel_configs:'
echo ' - source_labels: [__address__]'
echo ' target_label: __param_target'
echo ' - source_labels: [__param_target]'
echo ' target_label: instance'
echo ' - target_label: __address__'
echo ' replacement: localhost:9115'
} >> $PYAML/prometheus.yml
cp $PYAML/blackbox.yml $PYAML/backups/
{
echo 'modules:'
echo ' http_2xx:'
echo ' prober: http'
echo ' timeout: 20s'
echo ' http:'
echo ' valid_status_codes: []'
echo ' valid_http_versions: ["HTTP/1.1", "HTTP/2"]'
echo ' ip_protocol_fallback: false'
echo ' method: GET'
echo ' follow_redirects: true'
echo ' preferred_ip_protocol: "ip4"'
echo ' fail_if_ssl: false'
echo ' fail_if_not_ssl: false'
echo ' tls_config:'
echo ' insecure_skip_verify: true'
echo ' #basic_auth:'
echo ' #username: "username"'
echo ' #password: "password"'
echo ' http_post_2xx:'
echo ' prober: http'
echo ' http:'
echo ' method: POST'
echo ' tcp_connect:'
echo ' prober: tcp'
echo ' pop3s_banner:'
echo ' prober: tcp'
echo ' tcp:'
echo ' query_response:'
echo ' - expect: "^+OK"'
echo ' tls: true'
echo ' tls_config:'
echo ' insecure_skip_verify: false'
echo ' grpc:'
echo ' prober: grpc'
echo ' grpc:'
echo ' tls: true'
echo ' preferred_ip_protocol: "ip4"'
echo ' grpc_plain:'
echo ' prober: grpc'
echo ' grpc:'
echo ' tls: false'
echo ' service: "service1"'
echo ' ssh_banner:'
echo ' prober: tcp'
echo ' tcp:'
echo ' query_response:'
echo ' - expect: "^SSH-2.0-"'
echo ' - send: "SSH-2.0-blackbox-ssh-check"'
echo ' irc_banner:'
echo ' prober: tcp'
echo ' tcp:'
echo ' query_response:'
echo ' - send: "NICK prober"'
echo ' - send: "USER prober prober prober :prober"'
echo ' - expect: "PING :([^ ]+)"'
echo ' send: "PONG '\$'{1}"'
echo ' - expect: "^:[^ ]+ 001"'
echo ' icmp:'
echo ' prober: icmp'
echo ' icmp_ttl5:'
echo ' prober: icmp'
echo ' timeout: 5s'
echo ' icmp:'
echo ' ttl: 5'
} > $PYAML/blackbox.yml
if ! blackbox_exporter --config.check --config.file $PYAML/blackbox.yml; then
echo "There's a problem with the blackbox_exporter.yml config file"
exit 2
else
systemctl restart blackbox_exporter
fi
### Not required used for my test machine ###
# firewall-cmd --add-port=9115/tcp
# firewall-cmd --add-port=9115/tcp --permanent
}
}
############################
### Install AlertManager ###
############################
install_alertmanager() {
{
cd /tmp || exit 2
curl -s https://api.github.com/repos/prometheus/alertmanager/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi -
tar -xvf alertmanager*.tar.gz
cd alertmanager-*/ || exit 2
mv amtool alertmanager /usr/local/bin
mv alertmanager.yml /etc/prometheus
chown -R prometheus. /etc/prometheus
chown prometheus. /usr/local/bin/alertmanager /usr/local/bin/amtool
mkdir /var/lib/alertmanager
chown prometheus. /var/lib/alertmanager
if [ "$OS" = red ]; then
if [ "$OSVER" = 8 ]; then
restorecon -rv /usr/local/bin/alertmanager
fi
fi
touch $psdir/alertmanager.service
{
echo '[Unit]'
echo 'Description=Prometheus AlertManager Service'
echo 'Wants=network-online.target'
echo 'After=network-online.target'
echo ''
echo '[Service]'
echo 'User=prometheus'
echo 'Group=prometheus'
echo 'Type=simple'
echo "ExecStart=/usr/local/bin/alertmanager \\"
echo " --config.file /etc/prometheus/alertmanager.yml \\"
echo " --storage.path /var/lib/alertmanager/ \\"
echo ' --cluster.advertise-address=0.0.0.0:9093'
echo ''
echo '[Install]'
echo 'WantedBy=multi-user.target'
} > $psdir/alertmanager.service
cp $PYAML/alertmanager.yml $PYAML/backups/
{
echo 'global:'
echo ' smtp_smarthost: '\''nlb-exch-imaps-smt-x28bvtlo2e4rs-60fe7830ff427ab2.elb.us-gov-west-1.amazonaws.com:25'\'''
echo ' smtp_from: '\''alertmanager@calormen.net'\'''
echo ' smtp_require_tls: false'
echo ''
echo 'templates:'
echo ' - '\''/etc/prometheus/templates/email_alert.tmpl'\'''
echo ''
echo 'route:'
echo ' receiver: email'
echo ''
echo 'receivers:'
echo ' - name: '\''email'\'''
echo ' email_configs:'
echo " - to: '$email'"
echo ' from: '\''prometheus@us.calormen.net'\'''
echo ' #html: '\''{{ template "email" .}}'\'''
echo ' send_resolved: true'
} > $PYAML/alertmanager.yml
touch $PYAML/templates/email_alerts.tmpl
{
echo '{{ define "email" }}'
echo ''
echo '<html>'
echo ' <head>'
echo ' <style type="text/css">'
echo ' table {'
echo ' font-family: verdana,arial,sans-serif;'
echo ' font-size:11px;'
echo ' color:#333333;'
echo ' border-width: 1px;'
echo ' border-color: #999999;'
echo ' border-collapse: collapse;'
echo ' }'
edho ' table th {'
echo ' background-color:#ff6961;'
echo ' border-width: 1px;'
echo ' padding: 8px;'
echo ' border-style: solid;'
echo ' border-color: #F54C44;'
echo ' }'
echo ' table td {'
echo ' border-width: 1px;'
echo ' padding: 8px;'
echo ' border-style: solid;'
echo ' border-color: #F54C44;'
echo ' text-align: right;'
echo ' }'
echo ' </style>'
echo ' </head>'
echo ' <body>'
echo ' <table border=1>'
echo ' <thead>'
echo ' <tr>'
echo ' <th>Alert name</th>'
echo ' <th>Host</th>'
echo ' <th>Summary</th>'
echo ' <th>Description</th>'
echo ' </tr>'
echo ' </thead>'
echo ''
echo ' <tbody>'
echo ' {{ range .Alerts }}'
echo ' <tr>'
echo ' <td>{{ .Labels.alertname }}</td>'
echo ' <td>{{ .Annotations.host }}</td>'
echo ' <td>{{ .Annotations.summary }}</td>'
echo ' <td>{{ .Annotations.description }}</td>'
echo ' </tr>'
echo ' {{ end }}'
echo ' </tbody>'
echo ''
echo ' </table>'
echo ' </body>'
echo '</html>'
echo ''
echo '{{end}}'
} > $PYAML/templates/email_alerts.tmpl
systemctl daemon-reload
systemctl --now enable alertmanager
}
}
#######################
### Install Grafana ###
#######################
install_grafana() {
{
if [ "$OS" = ubuntu ]; then
$pkgmgr install -y apt-transport-https software-properties-common
mkdir -p /etc/apt/keyrings/
wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/grafana.gpg > /dev/null
echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" | sudo tee -a /etc/apt/sources.list.d/grafana.list
$pkgmgr update
$pkgmgr install grafana
else
touch /etc/yum.repos.d/grafana.repo
{
echo '[grafana]'
echo 'name=grafana'
echo 'baseurl=https://packages.grafana.com/oss/rpm'
echo 'repo_gpgcheck=1'
echo 'enabled=1'
echo 'gpgcheck=1'
echo 'gpgkey=https://packages.grafana.com/gpg.key'
echo 'sslverify=1'
echo 'sslcacert=/etc/pki/tls/certs/ca-bundle.crt'
} > /etc/yum.repos.d/grafana.repo
dnf -y repolist
dnf -y install grafana
fi
systemctl daemon-reload
systemctl enable --now grafana-server
### Not required used for my test machine ###
# firewall-cmd --add-port=3000/tcp
# firewall-cmd --add-port=3000/tcp --permanent
}
}
install_nginx() {
{
$pkgmgr install nginx
sitesa=/etc/nginx/sites-available
sitese=/etc/nginx/sites-enabled/
touch $sitesa/prometheus.conf
{
echo 'server {'
echo ' listen 80;'
echo ' listen [::]:80;'
echo ''
echo " server_name prometheus.$domain;"
echo ''
echo ' location / {'
echo ' proxy_pass http://localhost:9090/;'
echo ' include proxy_params;'
echo ' }'
echo '}'
} > $sitesa/prometheus.conf
touch $sitesa/metrics.conf
{
echo 'server {'
echo ' listen 80;'
echo ' listen [::]:80;'
echo ''
echo " server_name metrics.$domain;"
echo ''
echo ' location / {'
echo ' proxy_pass http://localhost:3000/;'
echo ' include proxy_params;'
echo ' }'
echo '}'
} > $sitesa/metrics.conf
touch $sitesa/alerts.conf
{
echo 'server {'
echo ' listen 80;'
echo ' listen [::]:80;'
echo ''
echo " server_name alerts.$domain;"
echo ''
echo ' location / {'
echo ' proxy_pass http://localhost:9093/;'
echo ' include proxy_params;'
echo ' }'
echo '}'
} > $sitesa/alerts.conf
touch $sitesa/loki.conf
{
echo 'server {'
echo ' listen 80;'
echo ' listen [::]:80;'
echo ''
echo " server_name loki.$domain;"
echo ''
echo ' location / {'
echo ' proxy_pass http://localhost:9093/;'
echo ' include proxy_params;'
echo ' }'
echo '}'
} > $sitesa/alerts.conf
ln -s $sitesa/prometheus.conf $sitese
ln -s $sitesa/metrics.conf $sitese
ln -s $sitesa/alerts.conf $sitese
ln -s $sitesa/prometheus.conf $sitese
systemctl restart nginx
}
}
install_certbot() {
{
systemctl stop nginx
if [ "$OS" = ubuntu ]; then
$pkgmgr update
$pkgmgr install python3-certbot-nginx certbot
systemctl enable --now certbot.timer
else
$pkgmgr install epel-release
$pkgmgr install certbot python3-certbot-nginx
touch /etc/cron.weekly/certbot
chmod +x /etc/cron.weekly/certbot
{
echo '#!/bin/sh'
echo 'certbot renew'
} > /etc/cron.weekly/certbot
fi
}
}
######################
### Function Calls ###
######################
install_prometheus
install_node_exporter
install_blackbox
install_alertmanager
install_grafana
install_nginx
#install_certbot