#! /bin/bash

#############################################################
#### Prometheus install Script for Oracle Linux,         #### 
#### Centos/Redhat and Debian/Ubuntu Servers.            ####       
####                                                     ####
#### Author: Phil Connor 08/27/2023                      ####
#### Contact: pconnor@ara.com                            ####
#### Version 2.01.100423                                 ####
####                                                     ####
#### To use this script chmod it to 755                  ####
#### or simply type bash <filename.sh>                   ####  
#############################################################

########################
#### User Variables ####
########################
domain=mylinux.work
email=phil@$domain
########################
### System Variables ###
########################
if [ "$(command -v lsb_release)" ]; then
    OS=$(lsb_release -i | awk '{print $3}' | tr '[:upper:]' '[:lower:]')
else
    OS=$(grep PRETTY_NAME /etc/os-release | sed 's/PRETTY_NAME=//g' | tr -d '="' | awk '{print $1}' | tr '[:upper:]' '[:lower:]')
fi

PYAML=/etc/prometheus

if ! [ -d "/usr/lib/systemd/system" ]; then
    psdir='/etc/systemd/system'
else
    psdir='/usr/lib/systemd/system'
fi

#########################
### Check permissions ###
#########################
if [[ $EUID -ne 0 ]]; then
    echo ''
    echo "This script must be run as root! Login as root, or sudo/su." 
    echo ''
    exit 1;
fi

######################
### Package Manager ##
######################
if [ "$OS" = ubuntu ]; then
    pkgmgr='apt -y'
else
    pkgmgr='dnf -y'
fi

###################################
#### Add Prometheus User/Group ####
###################################
if ! grep prometheus /etc/passwd; then 
    groupadd --system prometheus
    if [ "$OS" = ubuntu ]; then
        useradd -s /sbin/nologin --system -g prometheus prometheus
    else
        useradd -m -s /bin/false prometheus -g prometheus
    fi
fi

#################################
#### Check for wget and curl #### 
#################################
if [ ! "$(command -v wget)" ]; then
    $pkgmgr install wget
fi

if [ ! "$(command -v curl)" ]; then
    $pkgmgr install curl
fi

##########################
### Install Prometheus ###
##########################
install_prometheus() {
    {
        mkdir /etc/prometheus
        mkdir /var/lib/prometheus
        chown prometheus /var/lib/prometheus/
        
        for i in backups rules templates consoles console_libraries
        do 
            mkdir -p /etc/prometheus/${i} 
            chown -R prometheus. /etc/prometheus/${i}
            chmod -R 755 /etc/prometheus/${i}
        done

        cd /tmp || exit 2
        curl -s https://api.github.com/repos/prometheus/prometheus/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi -
        tar -xvf prometheus*.tar.gz
        cd prometheus-*/ || exit 2
        mv prometheus promtool /usr/local/bin/
        chown prometheus. /usr/local/bin/prometheus /usr/local/bin/promtool
        mv prometheus.yml /etc/prometheus/
        mv consoles/ console_libraries/ /etc/prometheus/
        chown -R prometheus. /var/lib/prometheus/

        if [ "$OS" = red ]; then
            if [ "$OSVER" = 8 ]; then
                restorecon -rv /usr/local/bin/node_exporter
            fi
        fi

        cp $PYAML/prometeus.yml $PYAML/backups/
        {
            echo '# Global config'
            echo 'global:'
            echo '  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. '
            echo '  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. '
            echo '  scrape_timeout: 15s  # scrape_timeout is set to the global default (10s).'
            echo ''
            echo '# Alertmanager configuration'
            echo 'alerting:'
            echo '  alertmanagers:'
            echo '  - static_configs:'
            echo '    - targets:'
            echo '      - alertmanager:9093'
            echo ''
            echo '# Load rules once and periodically evaluate them according to the global '\''evaluation_interval'\''.'
            echo 'rule_files:'
            echo '# - "first_rules.yml"'
            echo '# - "second_rules.yml"'
            echo ''
            echo '# A scrape configuration containing exactly one endpoint to scrape:# Here it'\''s Prometheus itself.'
            echo 'scrape_configs:'
            echo '  # The job name is added as a label '\''job=<job_name>'\'' to any timeseries scraped from this config.'
            echo '  - job_name: '\''prometheus'\'''
            echo ''
            echo '    # metrics_path defaults to '/metrics''
            echo '    # scheme defaults to '\''http'\''.'
            echo ''
            echo '    static_configs:'
            echo '      - targets: ['\''localhost:9090'\'']'
            echo ''
            echo '  - job_name: '\''server_metrics'\'''
            echo '    scrape_interval: 5s'
            echo '    static_configs:'
            echo '      - targets: ['\''localhost:9100'\'']'
            echo '        labels:'
            echo '          alias: Prometheus Server'
        } > /etc/prometheus/prometheus.yml

        ### Not required used for my test machine ###
        # firewall-cmd --add-port=9090/tcp
        # firewall-cmd --add-port=9090/tcp --permanent
        # 
        
        touch $psdir/prometheus.service

        {
            echo '[Unit]'
            echo 'Description=Prometheus Time Series Collection and Processing Server'
            echo 'Documentation=https://prometheus.io/docs/introduction/overview/'
            echo 'Wants=network-online.target'
            echo 'After=network-online.target'
            echo ''
            echo '[Service]'
            echo 'Type=simple'
            echo 'User=prometheus'
            echo 'Group=prometheus'
            echo ''
            echo "ExecReload=/bin/kill -HUP \$MAINPID"
            echo "ExecStart=/usr/local/bin/prometheus \\"
            echo "    --config.file /etc/prometheus/prometheus.yml \\"
            echo "    --storage.tsdb.path /var/lib/prometheus/data \\"
            echo "    --web.console.templates=/etc/prometheus/consoles \\"
            echo "    --web.console.libraries=/etc/prometheus/console_libraries \\"
            echo "    --web.listen-address=0.0.0.0:9090 \\"
            echo "    --web.external-url= \\"
            echo '    --enable-feature=new-service-discovery-manager,exemplar-storage,extra-scrape-metrics'
            echo ''
            echo 'Restart=always'
            echo 'RestartSec=5s'
            if [ "$OS" = ubuntu ]; then
                echo 'SyslogIdentifier=prometheus'
            fi
            echo 'Restart=always'
            echo ''
            echo '[Install]'
            echo 'WantedBy=multi-user.target'
        } > $psdir/prometheus.service
            
        systemctl daemon-reload
        systemctl enable --now prometheus
        
    }
}

#############################
### Install node_exporter ###
#############################
install_node_exporter() {
    {
        cd /tmp || exit 2
        curl -s https://api.github.com/repos/prometheus/node_exporter/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi -
        tar -xvf node_exporter*.tar.gz
        cd node_exporter-*/ || exit 2
        mv node_exporter /usr/local/bin
        chown prometheus. /usr/local/bin/node_exporter
        
        if [ "$OS" = red ]; then
            if [ "$OSVER" = 8 ]; then
                restorecon -rv /usr/local/bin/node_exporter
            fi
        fi

        touch $psdir/node_exporter.service
        {
            echo '[Unit]'
            echo 'Description=Prometheus Node Exporter'
            echo 'Wants=network-online.target'
            echo 'After=network-online.target'
            echo ''
            echo '[Service]'
            echo 'User=prometheus'
            echo 'Group=prometheus'
            echo 'Type=simple'
            echo "ExecStart=/usr/local/bin/node_exporter \\"
            echo "    --collector.ethtool \\"
            echo "    --collector.interrupts \\"
            echo "    --collector.processes \\"
            echo "    --collector.systemd \\"
            echo '    --collector.tcpstat'
            echo ''
            echo '[Install]'
            echo 'WantedBy=multi-user.target'
        } > $psdir/node_exporter.service

        systemctl daemon-reload
        systemctl enable --now node_exporter
        
        ### Not required used for my test machine ###
        # firewall-cmd --add-port=9100/tcp
        # firewall-cmd --add-port=9100/tcp  --permanent

        systemctl restart prometheus
    }
}

########################
### Install BlackBox ###
########################
install_blackbox() {
    {
        cd /tmp || exit 2
        curl -s https://api.github.com/repos/prometheus/blackbox_exporter/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi -
        tar -xvf blackbox_exporter*.tar.gz
        cd blackbox_exporter-*/ || exit 2
        mv blackbox_exporter /usr/local/bin
        chown prometheus. /usr/local/bin/blackbox_exporter
        mv  blackbox.yml /etc/prometheus/
        chown -R prometheus. /etc/prometheus/

        if [ "$OS" = red ]; then
            if [ "$OSVER" = 8 ]; then
                restorecon -rv /usr/local/bin/node_exporter
            fi
        fi

        touch $psdir/blackbox_exporter.service
        
        {
            echo '[Unit]'
            echo 'Description=Prometheus Blackbox Exporter Http/Https Montoring'
            echo 'After=network.target'
            echo ''
            echo '[Service]'
            echo 'User=prometheus'
            echo 'Group=prometheus'
            echo 'Type=simple'
            echo "ExecStart=/usr/local/bin/blackbox_exporter \\"
            echo "    --config.file /etc/prometheus/blackbox.yml \\"
            echo '    --web.listen-address=":9115"'
            echo ''
            echo 'Restart=always'
            echo ''
            echo '[Install]'
            echo 'WantedBy=multi-user.target'
        } > $psdir/blackbox_exporter.service

        systemctl daemon-reload
        systemctl enable --now blackbox_exporter

        {
            echo '  - job_name: '"'blackbox'"''
            echo '    metrics_path: /probe'
            echo '    params:'
            echo '      module: [http_2xx]' 
            echo '    static_configs:'
            echo '      - targets:'
            echo '      #### Local Targets ####'
            echo '         - http://localhost:9090'
            echo ''
            echo '      #### Remote Targets ####'
            echo '         #- https://google.com'
            echo ''
            echo '    relabel_configs:'
            echo '     - source_labels: [__address__]'
            echo '       target_label: __param_target'
            echo '     - source_labels: [__param_target]'
            echo '       target_label: instance'
            echo '     - target_label: __address__'
            echo '       replacement: localhost:9115'
        } >> $PYAML/prometheus.yml
    
        cp $PYAML/blackbox.yml $PYAML/backups/
        
        {
            echo 'modules:'
            echo '  http_2xx:'
            echo '    prober: http'
            echo '    timeout: 20s'
            echo '    http:'
            echo '      valid_status_codes: []'
            echo '      valid_http_versions: ["HTTP/1.1", "HTTP/2"]'
            echo '      ip_protocol_fallback: false'
            echo '      method: GET'
            echo '      follow_redirects: true'
            echo '      preferred_ip_protocol: "ip4"'
            echo '      fail_if_ssl: false'
            echo '      fail_if_not_ssl: false'
            echo '      tls_config:'
            echo '        insecure_skip_verify: true'
            echo '      #basic_auth:'
            echo '         #username: "username"'
            echo '         #password: "password"'
            echo '  http_post_2xx:'
            echo '    prober: http'
            echo '    http:'
            echo '      method: POST'
            echo '  tcp_connect:'
            echo '    prober: tcp'
            echo '  pop3s_banner:'
            echo '    prober: tcp'
            echo '    tcp:'
            echo '      query_response:'
            echo '      - expect: "^+OK"'
            echo '      tls: true'
            echo '      tls_config:'
            echo '        insecure_skip_verify: false'
            echo '  grpc:'
            echo '    prober: grpc'
            echo '    grpc:'
            echo '      tls: true'
            echo '      preferred_ip_protocol: "ip4"'
            echo '  grpc_plain:'
            echo '    prober: grpc'
            echo '    grpc:'
            echo '      tls: false'
            echo '      service: "service1"'
            echo '  ssh_banner:'
            echo '    prober: tcp'
            echo '    tcp:'
            echo '      query_response:'
            echo '      - expect: "^SSH-2.0-"'
            echo '      - send: "SSH-2.0-blackbox-ssh-check"'
            echo '  irc_banner:'
            echo '    prober: tcp'
            echo '    tcp:'
            echo '      query_response:'
            echo '      - send: "NICK prober"'
            echo '      - send: "USER prober prober prober :prober"'
            echo '      - expect: "PING :([^ ]+)"'
            echo '        send: "PONG '\$'{1}"'
            echo '      - expect: "^:[^ ]+ 001"'
            echo '  icmp:'
            echo '    prober: icmp'
            echo '  icmp_ttl5:'
            echo '    prober: icmp'
            echo '    timeout: 5s'
            echo '    icmp:'
            echo '      ttl: 5'
        } > $PYAML/blackbox.yml
        
        if ! blackbox_exporter --config.check --config.file $PYAML/blackbox.yml; then
            echo "There's a problem with the blackbox_exporter.yml config file" 
            exit 2
        else 
            systemctl restart blackbox_exporter
        fi

        ### Not required used for my test machine ###
        # firewall-cmd --add-port=9115/tcp
        # firewall-cmd --add-port=9115/tcp --permanent
    }
}

############################
### Install AlertManager ###
############################
install_alertmanager() {
    {
        cd /tmp || exit 2
        curl -s https://api.github.com/repos/prometheus/alertmanager/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi -
        tar -xvf alertmanager*.tar.gz
        cd alertmanager-*/ || exit 2
        mv amtool alertmanager /usr/local/bin
        mv  alertmanager.yml /etc/prometheus
        chown -R prometheus. /etc/prometheus
        chown prometheus. /usr/local/bin/alertmanager /usr/local/bin/amtool
        mkdir /var/lib/alertmanager
        chown prometheus. /var/lib/alertmanager

        if [ "$OS" = red ]; then
            if [ "$OSVER" = 8 ]; then
                restorecon -rv /usr/local/bin/alertmanager
            fi
        fi

        touch $psdir/alertmanager.service
        
        {
        echo '[Unit]'
        echo 'Description=Prometheus AlertManager Service'
        echo 'Wants=network-online.target'
        echo 'After=network-online.target'
        echo ''
        echo '[Service]'
        echo 'User=prometheus'
        echo 'Group=prometheus'
        echo 'Type=simple'
        echo "ExecStart=/usr/local/bin/alertmanager \\"
        echo "    --config.file /etc/prometheus/alertmanager.yml \\"
        echo "    --storage.path /var/lib/alertmanager/ \\"
        echo '    --cluster.advertise-address=0.0.0.0:9093'
        echo ''
        echo '[Install]'
        echo 'WantedBy=multi-user.target'
        } > $psdir/alertmanager.service

        cp $PYAML/alertmanager.yml $PYAML/backups/

        {
            echo 'global:'
            echo '  smtp_smarthost: '\''nlb-exch-imaps-smt-x28bvtlo2e4rs-60fe7830ff427ab2.elb.us-gov-west-1.amazonaws.com:25'\'''
            echo '   smtp_from: '\''alertmanager@calormen.net'\'''
            echo '   smtp_require_tls: false'
            echo ''
            echo 'templates:'
            echo '  - '\''/etc/prometheus/templates/email_alert.tmpl'\'''
            echo ''
            echo 'route:'
            echo '  receiver: email'
            echo ''
            echo 'receivers:'
            echo '  - name: '\''email'\'''
            echo '    email_configs:'
            echo "      - to: '$email'"
            echo '        from: '\''prometheus@us.calormen.net'\'''
            echo '        #html: '\''{{ template "email" .}}'\'''
            echo '        send_resolved: true'
        } > $PYAML/alertmanager.yml

        touch $PYAML/templates/email_alerts.tmpl

        {
            echo '{{ define "email" }}'
            echo ''
            echo '<html>'
            echo '   <head>'
            echo '      <style type="text/css">'
            echo '         table {'
            echo '         font-family: verdana,arial,sans-serif;'
            echo '         font-size:11px;'
            echo '         color:#333333;'
            echo '         border-width: 1px;'
            echo '         border-color: #999999;'
            echo '         border-collapse: collapse;'
            echo '         }'
            edho '         table th {'
            echo '         background-color:#ff6961;'
            echo '         border-width: 1px;'
            echo '         padding: 8px;'
            echo '         border-style: solid;'
            echo '         border-color: #F54C44;'
            echo '         }'
            echo '         table td {'
            echo '         border-width: 1px;'
            echo '         padding: 8px;'
            echo '         border-style: solid;'
            echo '         border-color: #F54C44;'
            echo '         text-align: right;'
            echo '         }'
            echo '      </style>'
            echo '   </head>'
            echo '   <body>'
            echo '      <table border=1>'
            echo '         <thead>'
            echo '           <tr>'
            echo '        <th>Alert name</th>'
            echo '        <th>Host</th>'
            echo '            <th>Summary</th>'
            echo '            <th>Description</th>'
            echo '           </tr>'
            echo '         </thead>'
            echo ''
            echo '         <tbody>'
            echo '       {{ range .Alerts }}'
            echo '            <tr>'
            echo '         <td>{{ .Labels.alertname }}</td>'
            echo '         <td>{{ .Annotations.host }}</td>'
            echo '         <td>{{ .Annotations.summary }}</td>'
            echo '         <td>{{ .Annotations.description }}</td>'
            echo '           </tr>'
            echo '          {{ end }}'
            echo '         </tbody>'
            echo ''
            echo '      </table>'
            echo '  </body>'
            echo '</html>'
            echo ''
            echo '{{end}}'
        } > $PYAML/templates/email_alerts.tmpl

        systemctl daemon-reload
        systemctl --now enable alertmanager

    }
}

#######################
### Install Grafana ###
#######################
install_grafana() {
    {
        if [ "$OS" = ubuntu ]; then 
            $pkgmgr install -y apt-transport-https software-properties-common
            mkdir -p /etc/apt/keyrings/
            wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/grafana.gpg > /dev/null
            echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" | sudo tee -a /etc/apt/sources.list.d/grafana.list
            $pkgmgr update
            $pkgmgr install grafana
        else
            touch /etc/yum.repos.d/grafana.repo
            {
                echo '[grafana]'
                echo 'name=grafana'
                echo 'baseurl=https://packages.grafana.com/oss/rpm'
                echo 'repo_gpgcheck=1'
                echo 'enabled=1'
                echo 'gpgcheck=1'
                echo 'gpgkey=https://packages.grafana.com/gpg.key'
                echo 'sslverify=1'
                echo 'sslcacert=/etc/pki/tls/certs/ca-bundle.crt'
            } > /etc/yum.repos.d/grafana.repo

            dnf -y repolist 
            dnf -y install grafana
        fi

        systemctl daemon-reload
        systemctl enable --now grafana-server

        ### Not required used for my test machine ###
        # firewall-cmd --add-port=3000/tcp 
        # firewall-cmd --add-port=3000/tcp --permanent
    }
}

install_nginx() {
    {
        $pkgmgr install nginx
        sitesa=/etc/nginx/sites-available
        sitese=/etc/nginx/sites-enabled/

        touch $sitesa/prometheus.conf
        {
            echo 'server {'
            echo '    listen 80;'
            echo '    listen [::]:80;'
            echo ''    
            echo "    server_name prometheus.$domain;"
            echo ''    
            echo '    location / {'
            echo '        proxy_pass http://localhost:9090/;'
            echo '        include proxy_params;'
            echo '        }'
            echo '}'
        } > $sitesa/prometheus.conf

        touch $sitesa/metrics.conf
        {
            echo 'server {'
            echo '    listen 80;'
            echo '    listen [::]:80;'
            echo ''    
            echo "    server_name metrics.$domain;"
            echo ''    
            echo '    location / {'
            echo '        proxy_pass http://localhost:3000/;'
            echo '        include proxy_params;'
            echo '        }'
            echo '}'
        } > $sitesa/metrics.conf

        touch $sitesa/alerts.conf
        {
            echo 'server {'
            echo '    listen 80;'
            echo '    listen [::]:80;'
            echo ''    
            echo "    server_name alerts.$domain;"
            echo ''    
            echo '    location / {'
            echo '        proxy_pass http://localhost:9093/;'
            echo '        include proxy_params;'
            echo '        }'
            echo '}'
        } > $sitesa/alerts.conf

                touch $sitesa/loki.conf
        {
            echo 'server {'
            echo '    listen 80;'
            echo '    listen [::]:80;'
            echo ''    
            echo "    server_name loki.$domain;"
            echo ''    
            echo '    location / {'
            echo '        proxy_pass http://localhost:9093/;'
            echo '        include proxy_params;'
            echo '        }'
            echo '}'
        } > $sitesa/alerts.conf
        
        ln -s $sitesa/prometheus.conf $sitese
        ln -s $sitesa/metrics.conf $sitese
        ln -s $sitesa/alerts.conf $sitese
        ln -s $sitesa/prometheus.conf $sitese
        
        systemctl restart nginx
    }
}

install_certbot() {
    {
        systemctl stop nginx
        if [ "$OS" = ubuntu ]; then
            $pkgmgr update
            $pkgmgr install python3-certbot-nginx certbot
            systemctl enable --now  certbot.timer
        else
            $pkgmgr install epel-release
            $pkgmgr install certbot python3-certbot-nginx
            touch /etc/cron.weekly/certbot
            chmod +x /etc/cron.weekly/certbot
                        {
                echo '#!/bin/sh'
                echo 'certbot renew'
            } > /etc/cron.weekly/certbot
        fi
    }
}

######################
### Function Calls ###
######################
install_prometheus
install_node_exporter
install_blackbox
install_alertmanager
install_grafana 
install_nginx
#install_certbot