#!/bin/bash
#
#   Copyright (C) 2017 Rackspace, Inc.
#
#   This program is free software; you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation; either version 2 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License along
#   with this program; if not, write to the Free Software Foundation, Inc.,
#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
#
#~ Usage: _tool_ [OPTION]
#~ Options:
#~   -h,--help       Print this help.
#~   -B,--backup     Copy the last log file to the backups dir.
#~   -S,--snapshot   Take a timestamped snapshot outside the regular rotation.
#~   -V,--version    Print version and exit.
#~

## Version
declare -r _VERSION='1.4.0'

## Default settings(can *NOT* be overriden in config file)
declare -r DATE=$( date +%F_%T )
declare -r LOCKFILE="/var/lock/recap.lock"
declare -r LOG_SUFFIX=$( date +%Y%m%d-%H%M%S )
PATH=/bin:/usr/bin:/sbin:/usr/sbin

## Default settings(modified through command line arguments)
declare -r default_BACKUP="no"
declare -r default_SNAPSHOT="no"
BACKUP="${default_BACKUP}"
SNAPSHOT="${default_SNAPSHOT}"

## Default settings(can be overridden in config file)
declare -r default_BACKUP_ITEMS="fdisk mysql netstat ps pstree resources"
declare -r default_BASEDIR="/var/log/recap"
declare -r default_RECAPLOG="${default_BASEDIR}/recap.log"
declare -r default_MAILTO=""
declare -r default_MIN_FREE_SPACE=0
declare -r default_USEFDISK="no"
declare -r default_USEPS="yes"
declare -r default_USEPSTREE="no"
declare -r default_USESLAB="no"
BACKUP_ITEMS="${default_BACKUP_ITEMS}"
BASEDIR="${default_BASEDIR}"
RECAPLOG="${BASEDIR}/recap.log"
MAILTO="${default_MAILTO}"
MIN_FREE_SPACE=${default_MIN_FREE_SPACE}
USEFDISK="${default_USEFDISK}"
USEPS="${default_USEPS}"
USEPSTREE="${default_USEPSTREE}"
USESLAB="${default_USESLAB}"

# Parent setting(other settings depend on this)
declare -r default_USERESOURCES="yes"
USERESOURCES="${default_USERESOURCES}"
# These depend on USERRESOURCES to be enabled("yes")
declare -r default_USEDF="yes"
declare -r default_USEFULLSTATUS="no"
declare -r default_USESAR="yes"
declare -r default_USESARQ="no"
declare -r default_USESARR="no"
USEDF="${default_USEDF}"
USEFULLSTATUS="${default_USEFULLSTATUS}"
USESAR="${default_USESAR}"
USESARQ="${default_USESARQ}"
USESARR="${default_USESARR}"

# Parent setting(other settings depend on this)
declare -r default_USENETSTAT="yes"
USENETSTAT="${default_USENETSTAT}"
# These depend on USENETSTAT to be enabled("yes")
declare -r default_USENETSTATSUM="no"
USENETSTATSUM="${default_USENETSTATSUM}"

# Parent setting(other settings depend on this)
declare -r default_USEMYSQL="no"
USEMYSQL="${default_USEMYSQL}"
# These depend on USEMYSQL to be enabled("yes")
declare -r default_DOTMYDOTCNF="/root/.my.cnf"
declare -r default_MYSQL_PROCESS_LIST="table"
declare -r default_USEINNODB="no"
declare -r default_USEMYSQLPROCESSLIST="no"
DOTMYDOTCNF="${default_DOTMYDOTCNF}"
MYSQL_PROCESS_LIST="${default_MYSQL_PROCESS_LIST}"
USEINNODB="${default_USEINNODB}"
USEMYSQLPROCESSLIST="${default_USEMYSQLPROCESSLIST}"

# Default command options(can be overriden in config file)
declare -r default_OPTS_LINKS="-dump"
declare -r default_OPTS_DF="-x nfs"
declare -r default_OPTS_FDISK="-l"
declare -r default_OPTS_FREE=""
declare -r default_OPTS_IOSTAT="-t -x 1 3"
declare -r default_OPTS_IOTOP="-b -o -t -n 3"
declare -r default_OPTS_NETSTAT="-atunp"
declare -r default_OPTS_NETSTAT_SUM="-a"
declare -r default_OPTS_PS="auxfww"
declare -r default_OPTS_PSTREE="-p"
declare -r default_OPTS_VMSTAT="-S M 1 3"
declare -r default_OPTS_STATUSURL="http://localhost:80/server-status"
OPTS_LINKS="${default_OPTS_LINKS}"
OPTS_DF="${default_OPTS_DF}"
OPTS_FDISK="${default_OPTS_FDISK}"
OPTS_FREE="${default_OPTS_FREE}"
OPTS_IOSTAT="${default_OPTS_IOSTAT}"
OPTS_IOTOP="${default_OPTS_IOTOP}"
OPTS_NETSTAT="${default_OPTS_NETSTAT}"
OPTS_NETSTAT_SUM="${default_OPTS_NETSTAT_SUM}"
OPTS_PS="${default_OPTS_PS}"
OPTS_PSTREE="${default_OPTS_PSTREE}"
OPTS_VMSTAT="${default_OPTS_VMSTAT}"
OPTS_STATUSURL="${default_OPTS_STATUSURL}"

# Internal variables
banner_start="--- Starting $( basename $0 )[$$] ---"
banner_end="--- Ending $( basename $0 )[$$] ---"

# Functions

# Timestamps
ts() {
  TS_FLAGS='--rfc-3339=seconds'
  date "${TS_FLAGS}"
}

# Logging messages
log() {
  # does not work in a while-loop as spawns a new shell
  local msg_type=$1
  shift
  local log_entry="$*"
  ## This avoids sending any output to stdout when executed through cron
  ## is helpful to avoid emails submitted, instead the logs contain the
  ## possible ERRORS
  if ! tty -s; then
    echo "$( ts ) [${msg_type}] ${log_entry}" 2>&1 >> "${RECAPLOG}"
    return 0
  fi
  if [[ "${VERBOSE}" ]]; then
    echo "$( ts ) [${msg_type}] ${log_entry}" 2>&1 | tee -a "${RECAPLOG}"
    return 0
  fi
  if [[ "${msg_type}" =~ "ERROR" ||
        "${msg_type}" =~ "WARNING" ]]; then
    echo "$( ts ) [${msg_type}] ${log_entry}" 2>&1 | tee -a "${RECAPLOG}"
  else
    echo "$( ts ) [${msg_type}] ${log_entry}" 2>&1 >> "${RECAPLOG}"
  fi
}

# Usage
print_usage() {
  grep -E '^#~' $0 | sed -e 's/^#~\s\?//' \
                         -e "s/_tool_/$( basename $0 )/"
}

# Cleanup function to remove lock
cleanup() {
  log INFO "$( basename $0 )[$$]: Caught signal - deleting ${LOCKFILE}"
  rm -f "${LOCKFILE}"
  log INFO "Execution time: ${SECONDS}s"
  log INFO "${banner_end}"
}

# Create a Lock so that recap does not try to run over itself.
recaplock() {
  (set -C; echo "$$" > "${LOCKFILE}") 2>/dev/null
  if [[ $? -ne 0 ]]; then
    log ERROR "$( basename $0 )[$$]: Lock File exists - exiting"
    exit 1
  else
    trap 'exit 2' 1 2 15 17 23
    trap 'cleanup' EXIT
    log INFO "$( basename $0 )[$$]: Created lock file: ${LOCKFILE}"
  fi
}

# Ensure our output directories exist before we start creating files
create_output_dirs() {
  BACKUPDIR="${BASEDIR}/backups"
  SNAPSHOTSDIR="${BASEDIR}/snapshots"
  local -a OUTPUT_DIRS
  OUTPUT_DIRS+=( "${BASEDIR}" )
  OUTPUT_DIRS+=( "${BACKUPDIR}" )
  OUTPUT_DIRS+=( "${SNAPSHOTSDIR}" )
  for OUTPUT_DIR in ${OUTPUT_DIRS[@]}; do
    if [[ ! -d "${OUTPUT_DIR}" ]]; then
      mkdir -p "${OUTPUT_DIR}"
    fi
  done
  chmod 0750 "${BASEDIR}"
}

# Create output file
create_output_file() {
  local OUTPUT_FILE="$1"
  if [[ -d "${OUTPUT_FILE}" ]]; then
    log ERROR "Target file already exists: ${OUTPUT_FILE}"
    exit
  else
    #print the data to the output file
    echo "${DATE}" > "${OUTPUT_FILE}"
  fi
}

# Check to see if the output directory exists
check_output_file() {
  local OUTPUT_FILE="$1"
  if [[ ! -r "${OUTPUT_FILE}" ]]; then
    log ERROR "The output file does not exist: ${OUTPUT_FILE}"
    exit
  fi
}

# Print output of "ps auxfww" to the specified file
print_ps() {
  log INFO "Starting 'ps' report"
  local LOGFILE="$1"
  ps ${OPTS_PS} 2>/dev/null >> "${LOGFILE}"
  log INFO "Ended 'ps' report"
}

# Print output of "pstree" to specified file
print_pstree() {
  log INFO "Starting 'pstree' report"
  local LOGFILE="$1"
  pstree ${OPTS_PSTREE} 2>/dev/null >> "${LOGFILE}"
  log INFO "Ended 'pstree' report"
}

# Print a blank line to the specified file
print_blankline() {
  local LOGFILE="$1"
  echo " " >> "${LOGFILE}"
}

# Print the output of "uptime" to the specified file
print_uptime() {
  log INFO "Starting 'uptime' report"
  local LOGFILE="$1"
  echo "UPTIME report" >> "${LOGFILE}"
  uptime 2>/dev/null >> "${LOGFILE}"
  log INFO "Ended 'uptime' report"
}

# Print the output of "free" to the specified file
print_free() {
  log INFO "Starting 'free' report"
  local LOGFILE="$1"
  echo "FREE report" >> "${LOGFILE}"
  free ${OPTS_FREE} 2>/dev/null >> "${LOGFILE}"
  log INFO "Ended 'free' report"
}

# Print the output of "vmstat" to the specified file
print_vmstat() {
  log INFO "Starting 'vmstat' report"
  local LOGFILE="$1"
  echo "VMSTAT report" >> "${LOGFILE}"
  vmstat ${OPTS_VMSTAT} 2>/dev/null >> "${LOGFILE}"
  log INFO "Ended 'vmstat' report"
}

# Print the output of "iostat" to the specified file
print_iostat() {
  log INFO "Starting 'iostat' report"
  local LOGFILE="$1"
  echo "IOSTAT report" >> "${LOGFILE}"
  iostat ${OPTS_IOSTAT} 2>/dev/null >> "${LOGFILE}"
  log INFO "Ended 'iostat' report"
}

# Print the output of "iotop" to the specified file
print_iotop() {
  log INFO "Starting 'iotop' report"
  local LOGFILE="$1"
  echo "IOTOP report" >> "${LOGFILE}"
  iotop ${OPTS_IOTOP} 2>/dev/null >> "${LOGFILE}"
  log INFO "Ended 'iotop' report"
}

# Print the output of sar to the specified file
print_sar() {
  log INFO "Starting 'sar' report"
  local LOGFILE="$1"
  local OPTION="$2"
  local FLAGS=''
  # check to see if we're going to use any parameters for sar
  if [[ "${OPTION}" == "r" ]]; then
    FLAGS="-r"
  elif [[ "${OPTION}" == "q" ]]; then
    FLAGS="-q"
  else
    FLAGS=""
  fi

  echo "SAR${FLAGS} report" >> "${LOGFILE}"
  sar "${FLAGS}" 2>/dev/null >> "${LOGFILE}"
  log INFO "Ended 'sar' report"
}

# Print the output of the web server status page to the specified file
print_http_fullstatus() {
  log INFO "Starting 'web status page' report"
  local LOGFILE="$1"
  echo "Web Status report" >> "${LOGFILE}"
  links ${OPTS_LINKS} "${OPTS_STATUSURL}" 2>/dev/null >> "${LOGFILE}"
  log INFO "Ended 'web status page' report"
}

# Print the output of "ss -atunp" to the specified file
print_netstat() {
  log INFO "Starting 'network socket' report"
  local LOGFILE="$1"
  echo "Network connections" >> "${LOGFILE}"
  if ss ${OPTS_NETSTAT} &>/dev/null; then
    ss ${OPTS_NETSTAT} 2>/dev/null >> "${LOGFILE}"
  else
    ss ${default_OPTS_NETSTAT} 2>/dev/null >> "${LOGFILE}"
    log WARNING "Bad config: '${OPTS_NETSTAT}', "\
                "using default: '${default_OPTS_NETSTAT}'"
  fi
  log INFO "Ended 'network socket' report"
}

# Print the output of "nstat -a" to the specified file
print_netstat_sum() {
  log INFO "Starting 'network summary' report"
  local LOGFILE="$1"
  echo "Network traffic summary" >> "${LOGFILE}"
  if nstat ${OPTS_NETSTAT_SUM} &>/dev/null; then
    nstat ${OPTS_NETSTAT_SUM} 2>/dev/null >> "${LOGFILE}"
  else
    nstat ${default_OPTS_NETSTAT_SUM} 2>/dev/null >> "${LOGFILE}"
    log WARNING "Bad config: '${OPTS_NETSTAT_SUM}', "\
                "using default: '${default_OPTS_NETSTAT_SUM}'"
  fi
  log INFO "Ended 'network summary' report"
}

# Print the output of "mysqladmin status" to the mysql file
print_mysql() {
  log INFO "Starting 'mysql status' report"
  local LOGFILE="$1"
  local MYCNF="$2"
  local PLESK_FILE="/etc/psa/.psa.shadow"
  if [[ -r "${PLESK_FILE}" ]]; then
    echo "MySQL (plesk) status" >> "${LOGFILE}"
    mysqladmin \
      -uadmin \
      -p$(cat "${PLESK_FILE}") \
      --connect-timeout=5 \
      status >> "${LOGFILE}"
  else
    echo "MySQL (${MYCNF}) status" >> "${LOGFILE}"
      mysqladmin \
        --defaults-file="${MYCNF}" \
        --connect-timeout=5 \
        status >> "${LOGFILE}"
  fi
  print_blankline "${LOGFILE}"
  log INFO "Ended 'mysql status' report"
}

# Print the non-truncated innodb status to the mysql file
print_mysql_innodb_status() {
  log INFO "Starting 'mysql innodb' report"
  local LOGFILE="$1"
  local MYCNF="$2"
  local mysql_cmd=''

  unset MYVALS PID_FILE TMPDIR
  if [[ -r "${PLESK_FILE}" ]]; then
    echo "MySQL (plesk) InnoDB status" >> "${LOGFILE}"
    mysql_cmd="mysql -uadmin -p$(cat ${PLESK_FILE}) --connect-timeout=5"
  else
    echo "MySQL (${MYCNF}) InnoDB status"  >> "${LOGFILE}"
    mysql_cmd="mysql --defaults-file=${MYCNF} --connect-timeout=5"
  fi
  # First, we establish which tmpdir and pid_file are in use, strip any
  # trailing slash and populate the file with current information.
  # We throw away the output of the "show engine innodb status" command as
  # it is likely being truncated, which we attempt to work around:
  local -a MYVALS=( $( IFS=$'\t' ${mysql_cmd} \
                         -Bse "SHOW VARIABLES LIKE 'pid_file'; \
                               SHOW VARIABLES LIKE 'tmpdir'; \
                               SHOW ENGINE INNODB STATUS;" \
                         2>/dev/null \
                       | head -n 2 ) )
  local PID_FILE="${MYVALS[1]%/}"
  local TMPDIR="${MYVALS[3]%/}"

  # Next, we grab a list of descriptors in the tmpdir:
  if [[ -r "${PID_FILE}" &&
        -d "${TMPDIR}" ]]; then
    for name in "/proc/"$(cat ${PID_FILE})"/fd/"*; do
      # We know that InnoDB's temporary files are always in the mysql tmpdir
      # and start with the string 'ib' followed by a randomly generated series
      # of characters.  Now we can compare the canonicalized path of fd to the
      # tmpdir/ib* pattern:
      if [[ "$(readlink -f "${name}")" == "${TMPDIR}"/ib* ]]; then
        # If any files match that pattern, we see if the first line in the file
        # starts with "===", which is the case in the particular InnoDB file we
        # care about:
        head -c8 "${name}" | grep -q '^====' && cat "${name}"  >> "${LOGFILE}"
      fi
    done
  fi
  log INFO "Ended 'mysql innodb' report"
}

# Print the output of "mysqladmin processlist" to the mysql file
print_mysql_procs() {
  log INFO "Starting 'mysql processlist' report"
  local LOGFILE="$1"
  local MYCNF="$2"
  local PLESK_FILE="/etc/psa/.psa.shadow"
  if [[ -r "${PLESK_FILE}" ]]; then
    echo "MySQL (plesk) processes" >> "${LOGFILE}"
    if [[ ${MYSQL_PROCESS_LIST,,} == "table" ]]; then
      mysqladmin \
        -uadmin \
        -p$( cat "${PLESK_FILE}" ) \
        -v processlist \
        --connect-timeout=5 \
        >> "${LOGFILE}"
    fi
    if [[ ${MYSQL_PROCESS_LIST,,} == "vertical" ]]; then
      mysql \
        -uadmin \
        -p$( cat "${PLESK_FILE}" ) \
        --connect-timeout=5 \
        -e "show full processlist\G" \
        >> "${LOGFILE}"
    fi
  else
    echo "MySQL (${MYCNF}) processes" >> "${LOGFILE}"
    if [[ ${MYSQL_PROCESS_LIST} == "table" ]]; then
      mysqladmin \
        --defaults-file="${MYCNF}" \
        -v processlist \
        --connect-timeout=5 \
        >> "${LOGFILE}"
    elif [[ ${MYSQL_PROCESS_LIST,,} == "vertical" ]]; then
      mysql \
        --defaults-file="${MYCNF}" \
        --connect-timeout=5 \
        -e "show full processlist\G" \
        >> "${LOGFILE}"
    fi
  fi
  print_blankline "${LOGFILE}"
  log INFO "Ended 'mysql processlist' report"
}

# Print the top 10 processes (by cpu usage) to the defined file
print_top_10_cpu() {
  log INFO "Starting 'top 10 cpu' report"
  local LOGFILE="$1"
  local pidstat_cpufield=0
  #We need to know whether the version of pidstat is > 10.1.4, since that added a UID field
  local version_strings="$( pidstat -V 2>&1 |
                              awk '{if(NR==1){print $NF}}' ) 10.1.4"
  local sorted_versions="$( echo ${version_strings} | tr ' ' '\n' |
                              sort --version-sort | tr '\n' ' ' )"
  if [[ "${version_strings}" == "${sorted_versions}" ]]; then
    pidstat_cpufield=6
  else
    pidstat_cpufield=7
  fi
  echo "Top 10 cpu using processes" >> "${LOGFILE}"
  # capture header
  pidstat 2>/dev/null | sed -n '3p' >> "${LOGFILE}"
  pidstat -l 2 2 2>/dev/null | grep -v '%system' |
    egrep ^Average: | sort -nr -k ${pidstat_cpufield} |
    head -11 >> "${LOGFILE}"
  log INFO "Ended 'top 10 cpu' report"
}

# Print the top 10 processes (by memory usage) to the defined file
print_top_10_mem() {
  log INFO "Starting 'top 10 memory' report"
  local LOGFILE="$1"
  echo "Top 10 memory using processes" >> "${LOGFILE}"
  ps auxww --sort=-rss 2>/dev/null | head -11 >> "${LOGFILE}"
  log INFO "Ended 'top 10 memory' report"
}

# Print the disk utilization to the defined file
print_df() {
  log INFO "Starting 'disk utilization' report"
  local LOGFILE="$1"
  local LOGFILE="$1"
  echo "Disk Utilization" >> "${LOGFILE}"
  df ${OPTS_DF} 2>/dev/null >> "${LOGFILE}"
  log INFO "Ended 'disk utilization' report"
}

# Print the disk partitions to the fdisk file
print_fdisk() {
  log INFO "Starting 'disk partition' report"
  local LOGFILE="$1"
  echo "Disk partitions" >> "${LOGFILE}"
  fdisk ${OPTS_FDISK} 2>&1 >> "${LOGFILE}"
  log INFO "Ended 'disk partition' report"
}

# Print the slabinfo command to the defined file
print_slabinfo() {
  log INFO "Starting 'slab info' report"
  local LOGFILE="$1"
  echo "Slab Information" >> "${LOGFILE}"
  printf "name\tactive\tnum_obj\tobj_size\thuh\n" >> "${LOGFILE}"
  tail -n+2 /proc/slabinfo 2>/dev/null |
    awk 'size=$3*$4 {print $1"\t"$2"\t"$3"\t"$4"\t"size/1048576}' |
    sort -k5gr >> "${LOGFILE}"
  log INFO "Ended 'slab info' report"
}

# Send the report via email
send_mail() {
  log INFO "Starting email report"
  # create a temp directory
  local TEMPDIR=$( mktemp -d "/tmp/rs.XXXXXXXXXX" )
  local REPORT="${TEMPDIR}/report.log"
  local ITEM_FILE=""
  local ITEM_USE=""

  # echo report files into temporary file for mailing
  for item in ${BACKUP_ITEMS}; do
    ITEM_FILE="${BASEDIR}/${item}_${LOG_SUFFIX}.log"
    # Build the variable to use and then use indirect ref to obtain its value
    ITEM_USE="USE${item^^}"
    if [[ -r "${ITEM_FILE}" && "${!ITEM_USE,,}" == "yes" ]]; then
      echo "-----BEGIN ${item^^} REPORT-----" >> "${REPORT}"
      cat "${ITEM_FILE}" >> "${REPORT}"
      echo "-----END ${item^^} REPORT-----" >> "${REPORT}"
      echo " " >> "${REPORT}"
    fi
  done
  log INFO "Ended email report"

  # send email summary to MAILTO address
  log INFO "Sending email report to: ${MAILTO}"
  if [[ -r "${REPORT}" ]]; then
    cat "${REPORT}" | mail -s "System Monitor Report - ${HOSTNAME}" "${MAILTO}"
    # Remove the temporary files
    rm -rf "${TEMPDIR}"
  fi
  log INFO "Sent email report"
}

# Manage item report
run_item_report() {
  local item="$1"
  local ITEM_FILE="${BASEDIR}/${item}_${LOG_SUFFIX}.log"

  if [[ "${SNAPSHOT,,}" == "yes" ]]; then
    ITEM_FILE="${SNAPSHOTSDIR}/${item}_${LOG_SUFFIX}.log_snapshot"
  fi

  create_output_file "${ITEM_FILE}"
  check_output_file "${ITEM_FILE}"
  eval "print_${item}" "${ITEM_FILE}"
}

# Manage resources report
run_resources_report() {
  local item="resources"
  local ITEM_FILE="${BASEDIR}/${item}_${LOG_SUFFIX}.log"

  if [[ "${SNAPSHOT,,}" == "yes" ]]; then
    ITEM_FILE="${SNAPSHOTSDIR}/${item}_${LOG_SUFFIX}.log_snapshot"
  fi

  create_output_file "${ITEM_FILE}"
  check_output_file "${ITEM_FILE}"
  print_uptime "${ITEM_FILE}"
  print_blankline "${ITEM_FILE}"
  print_free "${ITEM_FILE}"
  print_blankline "${ITEM_FILE}"
  print_vmstat "${ITEM_FILE}"
  print_blankline "${ITEM_FILE}"
  print_iostat "${ITEM_FILE}"
  print_blankline "${ITEM_FILE}"
  print_iotop "${ITEM_FILE}"
  print_blankline "${ITEM_FILE}"

  # check to see if sar should be run
  if [[ "${USESAR,,}" == "yes" ]]; then
    print_blankline "${ITEM_FILE}"
    print_sar "${ITEM_FILE}"
  fi

  # check to see if sar -r should be run
  if [[ "${USESARR,,}" == "yes" ]]; then
    # send sar -r output to output file
    print_blankline "${ITEM_FILE}"
    print_sar "${ITEM_FILE}" "r"
  fi

  # check to see if sar -q should be run
  if [[ "${USESARQ,,}" == "yes" ]]; then
    # send sar -q output to output file
    print_blankline "${ITEM_FILE}"
    print_sar "${ITEM_FILE}" "q"
  fi

  # check to see if webserver fullstatus should be run
  type -p 'links' > /dev/null
  if [[ $? -eq 0 && "${USEFULLSTATUS,,}" == "yes" ]]; then
    print_blankline "${ITEM_FILE}"
    print_http_fullstatus "${ITEM_FILE}"
  fi

  if [[ "${USEDF,,}" == "yes" ]]; then
    # send df -h output to output file
    print_blankline "${ITEM_FILE}"
    print_df "${ITEM_FILE}"
  fi

  if [[ "${USESLAB,,}" == "yes" ]]; then
    # send slabinfo output to output file
    print_blankline "${ITEM_FILE}"
    print_slabinfo "${ITEM_FILE}"
  fi

  print_blankline "${ITEM_FILE}"
  print_top_10_cpu "${ITEM_FILE}"
  print_blankline "${ITEM_FILE}"
  print_top_10_mem "${ITEM_FILE}"
}

# Manage netstat report
run_netstat_report() {
  local item="netstat"
  local ITEM_FILE="${BASEDIR}/${item}_${LOG_SUFFIX}.log"

  if [[ "${SNAPSHOT,,}" == "yes" ]]; then
    ITEM_FILE="${SNAPSHOTSDIR}/${item}_${LOG_SUFFIX}.log_snapshot"
  fi

  create_output_file "${ITEM_FILE}"
  check_output_file "${ITEM_FILE}"
  print_netstat "${ITEM_FILE}"

  # check to see if optional netstat summary report should be run
  if [[ "${USENETSTATSUM,,}" == "yes" ]]; then
    print_blankline "${ITEM_FILE}"
    print_netstat_sum "${ITEM_FILE}"
  fi
}

# Manage mysql report
run_mysql_report() {
  local item="mysql"
  local ITEM_FILE="${BASEDIR}/${item}_${LOG_SUFFIX}.log"

  if [[ "${SNAPSHOT,,}" == "yes" ]]; then
    ITEM_FILE="${SNAPSHOTSDIR}/${item}_${LOG_SUFFIX}.log_snapshot"
  fi

  create_output_file "${ITEM_FILE}"
  check_output_file "${ITEM_FILE}"
  log INFO "Starting iteration of mysql report(s)"
  # Iterate through the list of DOTMYDOTCNF config files
  local -a MYCNFS=( ${DOTMYDOTCNF//,/ } )
  for MYCNF in ${MYCNFS[@]}; do
    # Don't run reports if can't read the config file
    if [[ ! -r "${MYCNF}" ]]; then
      log ERROR "Unable to read: '${MYCNF}' in the 'DOTMYDOTCNF' config."
      continue
    fi
    # Don't run reports if can't connect to the instance
    if ! mysqladmin \
           --defaults-file="${MYCNF}" \
           --connect-timeout=5 \
           ping &>/dev/null; then
      log ERROR "Unable to connect using '${MYCNF}' in the 'DOTMYDOTCNF'"\
                "config."
      continue
    fi

    print_mysql "${ITEM_FILE}" "${MYCNF}"

    # check to see if the optional mysql process list should be generated
    if [[ "${USEMYSQLPROCESSLIST,,}" == "yes" ]]; then
      print_blankline "${ITEM_FILE}"
      print_mysql_procs "${ITEM_FILE}" "${MYCNF}"
    fi
    if [[ "${USEINNODB,,}" == "yes" ]]; then
      # send df -h output to output file
      print_blankline "${ITEM_FILE}"
      print_mysql_innodb_status "${ITEM_FILE}" "${MYCNF}"
    fi
  done
  log INFO "Ended iteration of mysql report(s)"
}

# Copy the last log file set to ${BACKUPDIR}
create_backup() {
  log INFO "Starting backup of reports"
  for log_file in ${BACKUP_ITEMS}; do
    ls -1t ${BASEDIR}/${log_file}_*.log 2>/dev/null |
    head -1 |
    xargs -I {} cp {} ${BACKUPDIR}
  done
  log INFO "Ended backup of reports"
}

# Check enough disk space is free
check_disk_space() {
  log INFO "Starting check for disk space"
  if [[ ${MIN_FREE_SPACE} -eq 0 ]]; then
    log INFO "Ended check for disk space"
    return 0
  else
    FREE_SPACE=$( df -PBM "${BASEDIR}" | awk '!/-blocks/ {print $4}' )
    FREE_SPACE=${FREE_SPACE%M}
    if [[ "${MIN_FREE_SPACE}" -ge "${FREE_SPACE}" ]]; then
      log ERROR "Unable to run recap due to not enough disk space"
    fi
  fi
  log INFO "Ended check for disk space"
}

## Main

# Evaluate input flags, print usage if more than one flag is used.
if [[ "$#" -gt 1 ]]; then
 log ERROR "Error: Only one flag is allowed"
 print_usage
 exit 1
fi

# If we only have one flag, see if we know how to handle it.
if [[ "$#" -gt 0 ]]; then
  case "$1" in
    -V|--version)
      echo "${_VERSION}"
      exit 0
      ;;
    -B|--backup)
      # backup latest log files
      BACKUP="yes"
      ;;
    -h|--help)
      # print usage
      print_usage
      exit 0
      ;;
    -S|--snapshot)
      # take a snapshot outside of the regular output rotation
      SNAPSHOT="yes"
      ;;
    *)
      # user entered an invalid flag, print warning and exit
      log ERROR "Invalid Input"
      print_usage
      exit 1
  esac
fi

# Verify that script is being run as root
if [[ "$(id -u)" != "0" ]]; then
  log ERROR "This script must be run as root."
  exit
fi

# Grab the server's host name
HOSTNAME="$( hostname )"

# Start logging
log INFO "${banner_start}"
log INFO "-- bash info: ${BASH_VERSINFO[@]}"

# Check for the configuration file.
# The following creates awareness about the old config location in
# /etc/recap, taking precendence the new config location /etc/recap.conf.
if [[ -r /etc/recap &&
      -r /etc/recap.conf ]]; then
  log WARNING "Configuration files exist at old(/etc/recap) and"\
              "new(/etc/recap.conf) locations. The file from the new"\
              "location will be used."
  log WARNING "Please move any missing configuration to /etc/recap.conf."
  source /etc/recap.conf
elif [[ -r /etc/recap &&
        ! -r /etc/recap.conf ]]; then
  log WARNING "Configuration file exists at old(/etc/recap) location."\
              "The file will be read."\
              "Please move your configuration file to /etc/recap.conf."\
              "Next version of recap will not read /etc/recap"
  source /etc/recap
elif [[ ! -r /etc/recap.conf ]]; then
  log WARNING "No configuration file found. Expecting /etc/recap.conf."
  log WARNING "Proceeding with defaults."
else
  source /etc/recap.conf
fi

# Create lock
recaplock

# Create output directory if it is not already present
create_output_dirs

# Check disk space before generating reports
check_disk_space

# Take a backup when needed
if [[ "${BACKUP,,}" == "yes" ]]; then
  log INFO "-- Taking backup, storing reports in ${BACKUPDIR}"
  create_backup
  exit 0
fi

# Log when running a snapshot
if [[ "${SNAPSHOT,,}" == "yes" ]]; then
  log INFO "-- Taking snapshot, storing reports in ${SNAPSHOTSDIR}"
fi

## Proceed to report generation
log INFO "-- Report suffix: ${LOG_SUFFIX}"

# Run the ps report
if [[ "${USEPS,,}" == "yes" ]]; then
  run_item_report "ps"
fi

# Run the resources report
#TODO: standardize the run_resources_report
if [[ "${USERESOURCES,,}" == "yes" ]]; then
  run_resources_report
fi

# Run the pstree report
if [[ "${USEPSTREE,,}" == "yes" ]]; then
  run_item_report "pstree"
fi

# Run the netstat report
#TODO: standardize the run_netstat_report
if [[ "${USENETSTAT,,}" == "yes" ]]; then
  run_netstat_report
fi

# Run the fdisk report
if [[ "${USEFDISK,,}" == "yes" ]]; then
  run_item_report "fdisk"
fi

# Run the mysql report
#TODO: standardize the run_mysql_report
if [[ "${USEMYSQL,,}" == "yes" ]]; then
  if type -p 'mysqladmin' > /dev/null; then
    run_mysql_report
  else
    log ERROR "mysql client is not installed, can't run mysql reports"
  fi
fi

# Check to see if report should be emailed
#TODO: Validate MAILTO format.
if [[ -n "${MAILTO}" ]]; then
  send_mail
fi

# We're done, time to exit
log INFO "Ended all the reports"
exit 0
