[Unit]
Description=Node Exporter
After=network.target


[Service]
User=nodeusr
Group=nodeusr
Type=simple
ExecStart=/usr/local/bin/node_exporter --collector.textfile.directory=/var/lib/node_exporter/textfile_collector --collector.systemd --collector.systemd.unit-include=(sshd|supervisor).service --collector.filesystem.ignored-mount-points=^/(sys|proc|dev|run|boot|host|etc)($|/) --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs|rootfs)$ --no-collector.hwmon --no-collector.nfsd

[Install]
WantedBy=multi-user.target

#!/bin/bash
#
# /etc/rc.d/init.d/node_exporter
#
#  Prometheus node exporter
#
#  description: Prometheus node exporter
#  processname: node_exporter

# Source function library.
. /etc/rc.d/init.d/functions

PROGNAME=node_exporter
PROG=/usr/local/bin/$PROGNAME
USER=nodeusr
LOGFILE=/var/log/node_exporter.log
LOCKFILE=/var/run/$PROGNAME.pid

start() {
    echo -n "Starting $PROGNAME: "
    cd /usr/local/bin/
    daemon --user $USER --pidfile="$LOCKFILE" "$PROG &>$LOGFILE &"
    echo $(pidofproc $PROGNAME) >$LOCKFILE
    echo
}

stop() {
    echo -n "Shutting down $PROGNAME: "
    killproc $PROGNAME
    rm -f $LOCKFILE
    echo
}


case "$1" in
    start)
    start
    ;;
    stop)
    stop
    ;;
    status)
    status $PROGNAME
    ;;
    restart)
    stop
    start
    ;;
    reload)
    echo "Sending SIGHUP to $PROGNAME"
    kill -SIGHUP $(pidofproc $PROGNAME)#!/bin/bash
    ;;
    *)
        echo "Usage: service node_exporter {start|stop|status|reload|restart}"
        exit 1
    ;;

#!/bin/bash

if [ "$USER" != "root" ];then
        echo "You must use <root> user to run..."
        exit 1
else
        echo "USER: ROOT"
fi


echo "INFO - check node_exporter if installed already"

service node_exporter status >> /dev/null

if [ $? -eq 0 ];then
  echo "Already installed, exit ..."
  exit 1
else
  echo "NOT install .. install now"
fi

command_exists() {
	command -v "$@" > /dev/null 2>&1
}

# check os version
# learn from https://get.docker.com
get_distribution() {

  if command_exists lsb_release; then
          dist_version="$(lsb_release --release | cut -f2)"
  fi
  if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then
          dist_version="$(. /etc/os-release && echo "$VERSION_ID")"
  fi
  echo "$dist_version"
}



do_install() {
  dist_version=$( get_distribution )

  useradd -rs /bin/false nodeusr
  mkdir -p /var/lib/node_exporter/textfile_collector
  chmod -R 777 /var/lib/node_exporter/
  yum install -y -q wget

  echo "install node_exporter ..."
  wget -q http://10.110.38.1/files/node_exporter
  mv node_exporter /usr/local/bin/
  chmod +x /usr/local/bin/node_exporter

  echo "install node_exporter directory-size.sh"
  wget -q http://10.110.38.1/files/directory-size.sh
  chmod +x directory-size.sh
  mv directory-size.sh /usr/local/bin/

  echo "System dist version: $dist_version"

  dist_version=$( echo "$dist_version" | cut -d'.' -f1)

  echo "install node_exporter service ..."
  if [ "$dist_version" == "7" ]; then

    wget -q http://10.110.38.1/files/node_exporter.service
    mv node_exporter.service /etc/systemd/system/

    systemctl daemon-reload
    systemctl start node_exporter
    systemctl enable node_exporter

  elif [ "$dist_version" == "6" ]; then

    wget -q http://10.110.38.1/files/node_exporter.rhel6.service
    mv node_exporter.rhel6.service /etc/init.d/node_exporter
    chmod +x /etc/init.d/node_exporter

    touch /var/log/node_exporter.log
    chmod 777 /var/log/node_exporter.log

    /etc/init.d/node_exporter start
    # auto start
    # chkconfig --add node_exporter
  
  else

    echo "Unsupport dist version"
    exit 0
  
  fi

}


do_install

[Unit]
Description=kafka_exporter
After=local-fs.target network-online.target network.target
Wants=local-fs.target network-online.target network.target

[Service]
ExecStart=/opt/kafka_exporter/kafka_exporter --kafka.server=x.x.x.x:6667
Restart=on-failure

[Install]
WantedBy=multi-user.target

- alert: HostOutOfDiskSpace
    expr: (node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0
    for: 10m
    labels:
      severity: critical
    annotations:
      summary: Host out of disk space (instance {{ $labels.instance }})
      description: "Disk is almost full (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: PrometheusTargetMissing
  expr: up == 0
  for: 5m
  labels:
    severity: critical
  annotations:
    summary: Prometheus target missing (instance {{ $labels.instance }})
    description: "A Prometheus target has disappeared. An exporter might be crashed.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: OutputErrorSizeAbove500G
  expr: node_directory_size_bytes{directory="/data/input/",job="canal_xxxx"}/1024/1024/1024 > 500
  for: 10m
  labels:
    team: node
    severity: critical
  annotations:
    summary: "Canal xxxx  Output Error Size High"
    description: '{{$labels.instance}}: Error Size: {{ $value | printf "%.2f" }})'

nohup karma --config.file /etc/prometheus/karma.yaml &

- id: alertmanager
  execute-command: "/home/<user>/alertmanager/sms.sh"
  command-working-directory: "/home/<user>/alertmanager/"

  pass-arguments-to-command:
  - source: payload
    name: status
  - source: payload
    name: alerts

#!/bin/bash

status=$1
alerts=$2

if [ "$status" == "firing" ];then
	status="!!警告!!"
else
	status="xx恢复xx"
fi

alertname=`echo $alerts | jq '.[0]["labels"]["alertname"]'`

title=`echo "[$status] $alertname"`
content=`echo $alerts | jq '.[]["annotations"]["description"]'`

sms=`echo -e "${title}\n${content}"`

phone='136xxxxxxxx'

curl -v -X POST http://ip:8768/sms/cmppSender -F phoneNumbers="$phone" -F "smContent=$sms"

./webhook -hooks hooks.yaml -verbose

[!!警告!!] "PrometheusTargetMissing"
"A Prometheus target has disappeared. An exporter might be crashed.
 VALUE = 0
 LABELS = map[__name__:up instance:sbi193:9100 job:canal_xxxxx]"

[xx恢复xx] "PrometheusTargetMissing"
"A Prometheus target has disappeared. An exporter might be crashed.
 VALUE = 0
 LABELS = map[__name__:up instance:sbi193:9100 job:canal_xxxx]"

{
  "version": "4",
  "groupKey": <string>,    // key identifying the group of alerts (e.g. to deduplicate)
  "status": "<resolved|firing>",
  "receiver": <string>,
  "groupLabels": <object>,
  "commonLabels": <object>,
  "commonAnnotations": <object>,
  "externalURL": <string>,  // backlink to the Alertmanager.
  "alerts": [
    {
      "labels": <object>,
      "annotations": <object>,
      "startsAt": "<rfc3339>",
      "endsAt": "<rfc3339>"
    }
  ]
}

{
  "version": "4",
  "groupKey": <string>,    // key identifying the group of alerts (e.g. to deduplicate)
  "status": "<resolved|firing>",
  "receiver": “web.hook",
  "groupLabels": {"alertname":"OutputErrorSize”}, // 对应 am 的 group by
  "commonLabels": <object>,
  "commonAnnotations":  {"description":"sbi163:9100: Output Error is above 500G (current value is: 1129.2498016357422)","summary":"sbi163:9100: Canal Worker Output Error High"},
  "externalURL": <string>,  // backlink to the Alertmanager.
  "alerts": [
{
  "annotations": {
    "summary": "/data/input/error/ size alert"
  },
  "endsAt": "0001-01-01T00:00:00Z",
  "fingerprint": "ac7d1a1af47b8b92",
  "generatorURL": "http://10.110.38.1:3000/d/WKkYjbiVk/canal_xxxxx?tab=alert&viewPanel=3&orgId=1",
  "labels": {
    "__name__": "node_directory_size_bytes",
    "alertname": "/data/input/error/ size alert",
    "directory": "/data/input/error/",
    "instance": "sbi139:9100",
    "job": "canal_xxxxx"
  },
  "startsAt": "2022-08-20T18:20:30Z",
  "status": "firing"
}
{
  "annotations": {
    "summary": "/data/input/error/ size alert"
  },
  "endsAt": "0001-01-01T00:00:00Z",
  "fingerprint": "2b0fb28362ff0526",
  "generatorURL": "http://10.110.38.1:3000/d/WKkYjbiVk/canal_xxxxx?tab=alert&viewPanel=3&orgId=1",
  "labels": {
    "__name__": "node_directory_size_bytes",
    "alertname": "/data/input/error/ size alert",
    "directory": "/data/input/error/",
    "instance": "sbi140:9100",
    "job": "canal_xxxxx"
  },
  "startsAt": "2022-08-20T18:20:30Z",
  "status": "firing"
}]}

#!/bin/bash

LockFile="/var/tmp/exporter.lock"

if [ -f $LockFile ];then
    echo "Compare time:"
    Time=`date +%s`
    LogTime=`stat -c %Y $LockFile`
    if [ $[$Time - $LogTime ] -lt 900 ];then
    echo "Another process is running."
    exit 1;
    else
    rm -f $LockFile;
    kill -15 `pgrep exporter.sh`;
    exit 1;
    fi
fi

touch $LockFile;

# make sure sub-dir put in front
/usr/local/bin/directory-size.sh /data/input/ /data/output/ > /tmp/metrics.prom.$$ && mv /tmp/metrics.prom.$$ /var/lib/node_exporter/textfile_collector/metrics.prom

rm -f $LockFile;
kill -15 `pgrep exporter.sh`;
exit 0;

#!/bin/sh
#
# Expose directory usage metrics, passed as an argument.
#
# Usage: add this to crontab:
#
# */5 * * * * prometheus directory-size.sh /var/lib/prometheus | sponge /var/lib/node_exporter/directory_size.prom
#
# sed pattern taken from https://www.robustperception.io/monitoring-directory-sizes-with-the-textfile-collector/
#
# Author: Antoine Beaupré <[email protected]>
echo "# HELP node_directory_size_bytes Disk space used by some directories"
echo "# TYPE node_directory_size_bytes gauge"
du --block-size=1 --summarize "$@" \
  | sed -ne 's/\\/\\\\/;s/"/\\"/g;s/^\([0-9]\+\)\t\(.*\)$/node_directory_size_bytes{directory="\2"} \1/p'

/usr/local/bin/directory-size.sh /data/input/ /data/output/ > /tmp/metrics.prom.$$ && mv /tmp/metrics.prom.$$ /var/lib/node_exporter/textfile_collector/metrics.prom

# HELP 指标名
# TYPE 指标名 指标类型
指标名{维度1="1", 维度2="2"} 值

#!/bin/sh
#
# Author: Meow-bot  <[email protected]>
LOG_PATH=/app/canal_xxx/log/
#DATA_MODEL=xxxx
day_id=`date +%Y%m%d`
#
#echo "# HELP canal_output_current_day_size_bytes canal output file size only current day"
#echo "# TYPE canal_output_current_day_size_bytes gauge"
#
#cat ${LOG_PATH}/info.log  | grep "${DATA_MODEL}_${day_id}" | awk -F '[:,]' 'BEGIN{sum=0}{sum+=$12}END{print "canal_output_current_day_size_bytes " sum}'
#
#echo "# HELP canal_output_size_bytes canal output file size"
#echo "# TYPE canal_output_size_bytes gauge"
#
#cat ${LOG_PATH}/info.log  | grep "${DATA_MODEL}_" | awk -F '[:,]' 'BEGIN{sum=0}{sum+=$12}END{print "canal_output_size_bytes " sum}'

echo "# HELP canal_error_log_size_bytes canal error log file size"
echo "# TYPE canal_error_log_size_bytes gauge"

ls -l ${LOG_PATH}/error.log  | awk  '{print "canal_error_log_size_bytes " $5}'

echo "# HELP canal_input_size_bytes canal input file size"
echo "# TYPE canal_input_size_bytes gauge"

du -s --block-size=1 /data/input/bak/${day_id} | awk '{print "canal_input_size_bytes " $1}'

/usr/local/bin/canal-size.sh > /tmp/canal.prom.$$ && mv /tmp/canal.prom.$$ /var/lib/node_exporter/textfile_collector/canal.prom