check_lysrdiff

#!/bin/sh
#
# This check script is maintained in a Git repository at
# https://git.lysator.liu.se/lysator/nagios-plugins.  Contact
# <ceder@lysator.liu.se> for commit access.

activity=26
stale=120

usage () {
  echo $0: usage: $0 '[ opt ] diskno partno' >&2
  echo options: >&2
  echo '  --activty N  Warn if no backup completed in the last N hours.' >&2
  echo '  --stale N    Critical if oldest backup is more than N hours old.' >&2
  exit 3
}

while [ $# -gt 2 ]
do
  case "x$1" in
    x--activity)
      activity=$2
      shift
      shift;;
    x--stale)
      stale=$2
      shift
      shift;;
    x--*)
      usage
      exit 1;;
    x*) break;;
  esac
done

if [ $# != 2 ]
then
  usage
fi

disk=$1
part=$2
BASE=/lysrdiff/$disk/perm/$part

# Emit a list of the first few backup jobs.  Keep the output small, by
# ending it with "and others" if it becomes too long.  ~200 characters
# are OK.
list_jobs()
{
    awk 'BEGIN { sz = 0 }
    	 NR > 1 && sz < 198 { printf ", "; sz += 2 }
	 sz < 200 { printf "%s", $3; sz += length($3) }
	 END { if (sz >= 200) printf ", others" } ' \
    | sed s'/, \([^,]*\)$/ and \1/'
}

if [ ! -f $BASE/lysrdiff.id ]
then
    echo CRITICAL - $BASE/lysrdiff.id: no such file
    exit 2
fi

if [ "`cat $BASE/lysrdiff.id`" != "$disk perm $part" ]
then
    echo CRITICAL - found `cat $BASE/lysrdiff.id` instead of $disk perm $part \
	>&2
    exit 2
fi

acttime=`date '+%Y-%m-%d %H:%M:%S' -d "$activity hours ago"`
staletime=`date '+%Y-%m-%d %H:%M:%S' -d "$stale hours ago"`

statecache=/tmp/lysrdiff-nagios-state
state=$statecache-$disk-$part

find $BASE/lysrdiff/state -type f \
	-printf "%TY-%Tm-%Td %TH:%TM:%TS %P\n" \
      | sort > $statecache.$disk.$part.$$
mv -f $statecache.$disk.$part.$$ $state || {
  echo UNKNOWN - mv failed
  rm $statecache.$disk.$part.$$
  exit 3
}
rm -f $state-stale $state-active || {
  echo UNKNOWN - rm failed
  rm $state
  exit 3
}

echo $staletime MARKER \
| sort $state - \
| grep -- '--start$\|MARKER' \
| sed -e '/MARKER$/,$d' \
> $state-stale

echo $acttime MARKER \
| sort $state - \
| grep -- '--end$\|MARKER' \
| sed -e '0,/MARKER$/d' \
> $state-active

trap "rm $state $state-stale $state-active" 0

FAILED=`grep -c -- '--fail$' $state`
STALE=`wc -l < $state-stale`
ACTIVITY=`wc -l < $state-active`

PERF="activity=$ACTIVITY stale=$STALE;1;10 failed=$FAILED;1;10"

worst=0
state_word=OK
message=""

set_state()
{
    if [ $1 -gt $worst ]
    then
	worst=$1
	state_word=$2
    fi

    if [ -n "$3" ]
    then
	if [ -z "$message" ]
	then
	    message="$3"
	else
	    message="$message $3"
	fi
    fi
}

if [ $FAILED -gt 0 ]
then
    msg="$FAILED failed backups: `sed -n 's/--fail$//p' $state | list_jobs`."
    if [ $FAILED -gt 10 ]
    then
	set_state 2 CRITICAL "$msg"
    else
	set_state 1 WARNING "$msg"
    fi
fi

if [ $STALE -gt 0 ]
then
    msg="$STALE stale backups: `sed 's/--start$//' $state-stale | list_jobs`."
    if [ $STALE -gt 10 ]
    then
	set_state 2 CRITICAL "$msg"
    else
	set_state 1 WARNING "$msg"
    fi
fi

if [ $ACTIVITY = 0 ]
then
  set_state 1 WARNING "No activity."
fi

if [ -f $BASE/lysrdiff/tasks ]
then
    TASKS=`wc -l < $BASE/lysrdiff/tasks`
    PERF="$PERF tasks=$TASKS"
    if [ $TASKS -gt $ACTIVITY ]
    then
	set_state 1 WARNING "$TASKS jobs defined, but only $ACTIVITY are active."
    fi
else
    set_state 2 CRITICAL "$BASE/lysrdiff/tasks is missing."
    PERF="$PERF tasks=0"
fi

if [ -z "$message" ]
then
    set_state 0 OK "all ok."
fi

echo "$state_word - $message | $PERF"
exit $worst