#!/bin/bash
# $Id: ssh_tunnel,v 2.11 2005/10/17 19:49:12 jlarsen Exp $
# Copyright 2004 and 2005 John R Larsen - theClaw56@larsen-family.us
# Free for personal use. Contact author for commercial use.
#
#------------------------------------------------------------------------------
# display_debug_help: Function to display debug help.
function display_debug_help () {
if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside display_debug_help" >> logfile; fi
cat << EOF | more
----- Debug Help ($SCRIPT_VER $SCRIPT_DATE) ---------------------------------------
This is a somewhat complex script involving multiple processes on multiple
machines. Debugging became difficult because of the amount of data. To make
it easier I developed the following mechanism. Debug is enabled in one of
three ways in decending order of precedence:
"-d 0xNNNN" on command line
SSH_TUNNEL_DEBUG_FLAGS environment variable
"debug_flags" file in same directory as ssh_tunnel
Debug output can be changed "on the fly" by changing the contents of "debug_flags",
which is read each time loader, tunnel, or pulse go through their while loops.
Each debug line is qualified with a construct like the following:
if [ \$((\$D & 0x1)) -ne 0 ]; then echo "[\$LINENO]\$PROGRAM_NAME:\$\$> Debug message or action"; fi
The expression on the left of -ne is a bash construct. The $((expr)) gets
evaluated and returns a value. The expression I'm using is "$D & 0xNN". The
& performs a bitwise AND and returns the result. If it is zero (no matching
bits) then the debug line is skipped. If it is non zero then the debug line
is performed. Each debug line can have multiple bits that turn it on or just
a single bit. The pattern "0xNN" is the collection of bits that turn the
debug on. The action of each bit is defined below.
00000000 - No debug
"Inside function name"
00000001 - Enable all the "Inside function name" debug lines
00000002 - Enable loader related "Inside function name" debug lines
00000004 - Enable tunnel related "Inside function name" debug lines
00000008 - Enable pulse related "Inside function name" debug lines
"loader"
00000010 - loader while loop message
00000020 - display of loader stats each time through its while loop
00000040 - check_server_heartbeat verbose output
00000080 - display loader startup progress messages
"tunnel"
00000100 - tunnel while loop message
00000200 - display of tunnel stats each time through its while loop
00000400 - display verbose ssh link connection
00000800 - display tunnel startup progress messages
"pulse"
00001000 - pulse while loop message
00002000 - display of pulse stats each time through its while loop
00004000 - check_client_heartbeat verbose output
00008000 - display pulse startup progress messages
"update_file"
00010000 - verbose ssh connection info when checking remote file info
00020000 - verbose scp copy info, timeout counter ticks, and killing subprocess
00040000 - remote and local file info
00080000 - "Inside update_file"
"Misc debug"
00100000 - test_pid verbose output
00200000 - process_command_line verbose output
00400000 - kill_ssh_link verbose output
00800000 - test_port verbose output
01000000 - sleep_time verbose output
02000000 - test_pid: Display ps output for dead PIDs
80000000 - Don't output the "Debug flags changed" messages in "set_debug_level"
Copyright: 2004 and 2005 by John R Larsen
Free for personal use. Contact the author for commercial use.
http://larsen-family.us theClaw56@larsen-family.us
EOF
} # display_debug_help
#------------------------------------------------------------------------------
# display_help: Function to display help
function display_help () {
if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside display_help" >> logfile; fi
cat << EOF | more
----- ssh_tunnel ($SCRIPT_VER $SCRIPT_DATE) --------------------------------------------
usage:
ssh_tunnel -p {loader | tunnel | pulse} [options]
This program is started by a cron job on an ssh client. It makes an ssh
connection with an ssh server and forwards ports as defined in "tunnel.conf". The
client and server are defined in required file loader.conf. All the files are
located in the same directory as ssh_tunnel, which must be invoked with the full
path. Considerable configuration is required. Use "ssh_tunnel -hs" to read.
-p loader - Runs on ssh client. Started by a cronjob, /etc/init.d/tunnel,
or manually. It launches "tunnel". It checks ssh server for
updated tunnel.conf and restarts "tunnel" when new file is found.
It does heartbeat verification on the client side and restarts
"tunnel" upon heartbeat failure or if "tunnel" pid is inactive.
-p tunnel - Runs on ssh client. Started by "loader". Establishes the ssh
tunnel with the ssh server using port forwardings defined in
tunnel.conf. It writes and copies client and server heartbeats.
It runs forever until killed by "loader" or kills itself if it
detects incorrect pid.
-p pulse - Runs on ssh server. Started by "tunnel" when it establishes the ssh
tunnel. It does heartbeat verification on the server side. It
runs forever until the tunnel is killed, heartbeat verification
fails, or it detects incorrect pid.
Options:
-c cronjob processing
-D n Daemon mode of operation and daemon sleep time in seconds
-d n debug level (ie. -d 0x4040 )
-dl delete logs
-ds delete statistics
-e adr email_address (If "none" then no email is sent)
(Separate multiple email addresses with commas and no spaces)
-h This help screen
-hd Help on debugging
-hs Help on configuration setup
-k kill a running instance of ssh_tunnel
-md Make an init.d daemon mode startup file. You MUST modify this for your situation!
-ml Make a loader.conf file. You MUST modify this for your situation!
-mt Make a tunnel.conf. You MUST modify this for your situation!
-mw Make a webpage.copy script. You MUST modify this for your situation!
-s show statistics
-st n sleep_time in seconds (Default: $SLEEP_TIME)
-wr n Web page update Rate in minutes
Note: This feature enabled if script webpage.copy exists in `pwd`.
The name of the generated web page (webpage.html) is passed
in \$1 to the script to allow renaming. webpage.copy must be stand
alone containing all steps needed to transfer the file, ie. scp or cp
Defaults: 10 minutes when tunnel is enabled
Each cron invocation when the tunnel is disabled
Copyright: 2004 and 2005 by John R Larsen
Free for personal use. Contact the author for commercial use.
http://larsen-family.us theClaw56@larsen-family.us
EOF
} # display_help
#------------------------------------------------------------------------------
# display_setup_help: Function to display help on ssh_tunnel configuration.
function display_setup_help () {
if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside display_setup_help" >> logfile; fi
cat << EOF | more
----- Configuration Help ($SCRIPT_VER $SCRIPT_DATE) ---------------------------------------
Before this tunnel will work there are certain configuration steps that must be done
on both the ssh client and ssh server machines.
1. The ssh client public key must be copied to the ~/.ssh/authorized_keys file
on the ssh server. These keys must not have a passphrase as that would require
entering a password, which can't be done in a cron script. The corresponding
private key must be on the client in the location specified in the loader.conf
file (see step 3).
2. All the other files go in the same directory as ssh_tunnel on both the client
and server. The directory name is important. It must have the name of the
server in it. For example, if the server's name is "linux-beast" then the path
on the client could be "/home/jlarsen/ssh_tunnel/linux-beast". "loader" uses grep
to search the output of "ps" for the ssh server's name. On cygwin clients the
the full path to the ssh_tunnel program must be no more than 48 characters, ie.
/home/jlarsen/ssh_tunnel/linux-beast/ssh_tunnel
Cygwin truncates redirected ps output to 80 characters. The server's name will
be lost if the absolute path is longer than 48 characters.
3. Create required file "loader.conf" in the same directory as ssh_tunnel on the client.
This file gets copied to and used by the server, but the controlled copy is on the
client. A sample of that file is given below. Modify the contents to match your
configuration. You can use "ssh_tunnel -ml" to create a template file to modify:
=~=~=~= Beginning of sample loader.conf =~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=
# Filename: loader.conf
# Description: This is the configuration file for the loader portion of the
# ssh_tunnel program. The contents of this file never change. This insures
# that the loader will always operate. It is important not to change the
# formatting of thse file because it is read by the ssh_tunnel and processed
# using awk which is expecting things to be in certain locations.
#
# loader.conf has two sections:
# 1. Configuration information such as the names of the ssh and sshd
# hosts, the working directories on each host, and other data as needed.
# 2. The ssh host configuration for the loader program is in this file.
# That configuration should never change. It needs to be at the end
# of the file because ssh uses everything between "Host" entries as
# configuration. There is only one Host section in this file.
#
########################################################################
# The following section contains host information
# SSH_CLIENT john-beast
# SSH_SERVER linux-beast
# SSH_SERVER_DIR /home/jlarsen/ssh_tunnel/john-beast
# EMAIL_ADDRESS none
#
########################################################################
# The following section is the ssh config file for the loader program.
Host loader
HostName linux-beast
Port = 2222
UserKnownHostsFile = /home/jlarsen/ssh_tunnel/linux-beast/known_hosts
User = jlarsen
IdentityFile = /home/jlarsen/ssh_tunnel/linux-beast/id_rsa.fhc-beast
=~=~=~= Ending of sample loader.conf =~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=
4. You must have a "known_hosts" file that has the host's identity in it.
The name and location of the known_hosts file is defined in the
config file. Without this file RSA authentication will fail.
The host key for "localhost" will be the same as the host key for
the ssh_tunnel host because it is really the same machine. Below is an
example of a known_hosts file. The key is really just oneline but has
been split up to appear in this document.
larsen-family.us,24.61.91.242,localhost ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIE
AxVOSV6Rvp9M7VkNxyKXTJOJIW81sggX+DH8e2bqGJjvrwuiQXFp+gXe7lF9ntcRb0RkjAqfa9+
OwMRQwMk+b4uxoHSE8k91FFo6yEmR9OXWEFqOUoo/c+2BlUfqDQqVAErpc3ZnMlPWvOqtNPiCG1
fEdWLxW6kEEfrpDk3LsOac=
5. Directory and file permissions are important. The RSA private key file must
only be readable by the owner of the file, ie. 400. The directory must only
be writeable by the owner, ie. 700. It's best to just set all the permissions
of the files so group and world can't access the directory or files.
6. Create required file "tunnel.conf" and put it on both the client and server
in the same directory as ssh_tunnel. The control copy of this file is on the server.
During tunnel operation if this file is modified on the server, it is copied by the
"loader" to the client, and the "loader" stops and restarts the "tunnel". This
makes it possible from the server side to make changes in the port forwardings,
email addresses, and email thresholds. The "loader" compares file sizes of the
remote and local versions of tunnel.conf. If the size changes then the file is
copied over. A sample of tunnel.conf is given below. Modify it to match your
configuration You can use "ssh_tunnel -mt" to create a template file to modify:
=~=~=~= Beginning of sample tunnel.conf =~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=
# Filename: tunnel.conf
#
# Description: This is the configuration file for the tunnel portion of the
# ssh_tunnel program. The contents of this file can be changed during program execution.
# It is important not to change the formatting of the file because it is read by the
# tunnel program and processed using awk which is expecting things to be in certain
# locations.
#
# tunnel.conf has two sections:
# 1. Configuration information such as email addresses and thresholds.
# 2. The ssh host configuration for the tunnel. This consists of two
# host definitions, tunnel and heartbeat. The tunnel is the permanent
# connection and defines port forwarding. The heartbeat uses one of
# the forwarded ports to write and copy heartbeat files between the
# ssh client and the ssh server. Changing heartbeats indicate that the
# tunnel is functioning.
#
########################################################################
# The following section contains configuration information
#
# The tunnel is either "enabled" or "disabled" based on the value given
# below. If disabled, then no processes are running on either the
# client or the server. A cron job on the client runs ssh_tunnel periodically
# to check if tunnel.conf has changed on the server. If a change is detected
# then the new tunnel.conf file is transfered over. If the tunnel state
# becomes "enabled" then the tunnel is activated.
# TUNNEL_STATE enabled
#
# The email addresses below receive diagnostic messages. Separate
# multiple addresses with commas and no white space
# EMAIL_ADDRESS none
#
# The threshold defined blow is how many failures in a row are required
# before an email is sent. This applies to failed port tests, failed
# ssh connections, and failed scp file transfers.
# EMAIL_THRESHOLD 4 (Number of failures before email is sent)
#
# The loader, tunnel, and pulse programs all have the same sleep value.
# The sleep time can be changed here. It must be in the range of
# 60 to 3600 seconds
# SLEEP_TIME 300
#
# The name below is used by the "pulse" program running on the server to detect
# if the IP address of the client has changed. pulse greps the output of
# netstat looking for this name. It sends an email if it changes. For this
# to work, the /etc/hosts file on the server must have the client's IP address
# associated with this name. Leave the name blank to turn this off.
# NETSTAT_CLIENT_NAME john-beast
#
########################################################################
# The following section is the ssh config file for the tunnel program.
Host ssh_tunnel
HostName linux-beast
Port = 2222
UserKnownHostsFile = /home/jlarsen/ssh_tunnel/linux-beast/known_hosts
User = jlarsen
IdentityFile = /home/jlarsen/ssh_tunnel/linux-beast/id_rsa.fhc-beast
Compression = yes
RemoteForward = 55920 cool-beast:5920
RemoteForward = 55925 awesome-beast:5925
RemoteForward = 55995 old-server:5995
RemoteForward = 50022 john-beast:22
LocalForward = 50022 linux-beast:22
########################################################################
Host heartbeat
HostName localhost
Port = 50022
UserKnownHostsFile = /home/jlarsen/ssh_tunnel/linux-beast/known_hosts
User = jlarsen
IdentityFile = /home/jlarsen/ssh_tunnel/linux-beast/id_rsa.fhc-beast
=~=~=~= Ending of sample tunnel.conf =~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=
7. To setup the tunnel first use ssh on the command line to verify that you can make
an ssh connection using the public/private keys without being required to enter a
password. Use the configuration file that ssh_tunnel will use like this:
ssh -F ./loader.conf loader
The above command will verify the settings in your loader.conf file. You might be warned
that the ssh server's key doesn't exist and you will be asked if you want to add it.
Answer yes. Once this key is in the known_hosts file you won't be asked for it anymore.
If the server's key gets changed somehow, you will need to remove the old key and replace
it with a new one.
The next step is to startup ssh_tunnel with debug enabled to display verbose output from
ssh connections and scp copies. Use "ssh_tunnel -hd" to see help about debugging. A
typical startup command line looks like this:
/home/jlarsen/ssh_tunnel/linux-beast/ssh_tunnel -d 0x40444 -p loader -st 65 &
The above line enables debugging, starts up the "loader" program, and sets sleeptime
to 65 seconds. The default sleep time is defined in tunnel.conf but command line
has precedence.
Once that works then setup a cron job to check that the tunnel is operating. A typical
crontab line looks like this:
0,30 * * * * /h/jlarsen/ssh_tunnel/linux-beast/ssh_tunnel -c
This will verify the "loader" is running every 30 minutes and restart it if the "loader"
pid isn't active.
8. Daemon operation. An alternate method to run ssh_tunnel is as a daemon that is started
by a script in /etc/init.d. You must have root privileges to do this. The startup script
"ssh_tunnel" can be made using the command "ssh_tunnel -md", which creates a script named
ssh_tunnel.init.d. Copy this script to /etc/init.d and rename it to ssh_tunnel. As root
execute the command "chkconfig --add ssh_tunnel". That will add the symbolic links to
the appropriate runtime levels. You can use "/etc/init.d/ssh_tunnel start" to start the
tunnel up. The default sleep time for the daemon is set to 300 seconds in this script.
Change the value as needed. Don't use daemon mode and cron job at the same time.
9. Webpage update. The "loader" and cron processing both look for the file "webpage.copy"
in the same directory as the ssh_tunnel script. If that file exists then then the file
webpage.html is generated and webpage.copy is invoked to copy webpage.html to a webserver.
The default update rate is about ten minutes. This can be changed using the -wr option
on the command line. Use the -mw option to generate a template webpage.copy script that
must be modified for your situation. If the tunnel state is "disabled" then the webpage.html
file is generated each time cron wakes ssh_tunnel up.
SUPPORTED PLATFORMS:
This script has been used successfully on the following systems:
Solaris 8 using bash v2.03.0(1)
Mandrake Linux 8.2 using bash v2.05.1(1)
Mandrake Linux 9.1 using bash v2.05b.0(1)
Cygwin 1.5.13 using bash v2.05b.0(1) hosted on Windows 2000 SP 4 and Windows XP SP 2
Redhat Workstation Enterprise 3
Redhat Workstation Enterprise 4 using bash v3.00.15(1)
Copyright: 2004 and 2005 by John R Larsen
Free for personal use. Contact the author for commercial use.
http://larsen-family.us theClaw56@larsen-family.us
EOF
} # display_setup_help
#------------------------------------------------------------------------------
# set_debug_level: Function that sets the $D variable based on a file, an
#environment variable, or a command line variable. See display_debug_help.
function set_debug_level () {
# Note: Command line -d overrides environment variable SSH_TUNNEL_DEBUG_FLAGS if it exists, which overrides debug_flags file if it exists
# Check one time if SSH_TUNNEL_DEBUG_FLAGS env variable exists and use it if it does
if [ "${ENV_VAR_TESTED:=FALSE}" == "FALSE" ]; then
ENV_VAR_TESTED=TRUE
D=${SSH_TUNNEL_DEBUG_FLAGS:-0}
if [ "$D" != "0" ] && [ $(($D & 0x80000000)) -ne 0 ]; then echo "[$LINENO]$$> ${PROGRAM_NAME:-startup}: set_debug_level: Debug flags changed to: $D" >> logfile; fi
fi
# Check the existence of a file named "debug_flags" and use it if conditions are right
if [ -e debug_flags ]; then
if [ "${DEBUG_FLAGS_FILE:=FIRST_READ}" = "FIRST_READ" ]; then
# Getting here means the debug_flags file has never been read.
DEBUG_FLAGS_FILE=`cat -s debug_flags`
if [ "$D" = "0" ]; then
# Getting here means $D was zero and the debug_flags contents should be used instead since env variable doesn't exist or has value of 0
D=$DEBUG_FLAGS_FILE
if [ "$D" != "0" ] && [ $(($D & 0x80000000)) -eq 0 ]; then echo "[$LINENO]$$> ${PROGRAM_NAME:-startup}: set_debug_level: Debug flags changed to: $D" >> logfile; fi
fi
else
# Read debug_flags file and update $D if the file has changed
NEW_DEBUG_FLAGS_FILE=`cat -s debug_flags`
if [ $NEW_DEBUG_FLAGS_FILE != $DEBUG_FLAGS_FILE ]; then
DEBUG_FLAGS_FILE=$NEW_DEBUG_FLAGS_FILE
D=$DEBUG_FLAGS_FILE
if [ $(($D & 0x80000000)) -eq 0 ]; then
echo "[$LINENO]$$> ${PROGRAM_NAME:-startup}: set_debug_level: Debug flags changed to: $D" >> logfile
fi
fi
fi
fi
} # set_debug_level
#-------------------------------------------------------------------------------
# get_version. Function that parses script version and date info from the RCS Id line.
function get_version () {
SCRIPT_VER="v`echo '$Id: ssh_tunnel,v 2.11 2005/10/17 19:49:12 jlarsen Exp $' | awk '{print $3}'`"
SCRIPT_DATE="`echo '$Id: ssh_tunnel,v 2.11 2005/10/17 19:49:12 jlarsen Exp $' | awk '{print $4}'`"
} # get_version
#------------------------------------------------------------------------------
# env_setup: Function that sets up the environment needed for the script
function env_setup () {
local func_name=env_setup
# Figure out which OS you're running on and setup program paths accordingly
OS_TYPE=`uname -a | awk '{print $3}'`
case $OS_TYPE in
5.8*) # Solaris 8
SSH=/opt/bin/ssh
export SCP=/opt/bin/scp
PS=/bin/ps
LS=/bin/ls
MAIL=/usr/ucb/mail
NETSTAT="/usr/bin/netstat -P tcp"
TELNET=/usr/bin/telnet
W=/usr/bin/w
ZIP_PGM=/usr/bin/zip
;;
2.4*) # Mandrake Linux 8.x and 9.x, Redhat Linux Work Station Enterprise 3
SSH=/usr/bin/ssh
export SCP=/usr/bin/scp
PS=/bin/ps
LS=/bin/ls
MAIL=/bin/mail
NETSTAT="/bin/netstat -t"
TELNET=/usr/bin/telnet
W=/usr/bin/w
ZIP_PGM=/usr/bin/zip
;;
2.6*) # Mandrake Linux 10.x, Redhat Linux Work Station Enterprise 4
SSH=/usr/bin/ssh
export SCP=/usr/bin/scp
PS=/bin/ps
LS=/bin/ls
MAIL=/bin/mail
NETSTAT="/bin/netstat -t"
TELNET=/usr/bin/telnet
W=/usr/bin/w
ZIP_PGM=/usr/bin/zip
;;
1.5*) # cygwin
# Base cygwin doesn't include many packages required for ssh_tunnel to work.
# Check that these packages have been installed in their default locations.
# Make sure the openssh package has been installed
if [ -e /usr/bin/ssh ]; then
SSH=/usr/bin/ssh
else
echo "ERROR [$LINENO]$func_name> Missing ssh. Install the openssh package." | tee -a logfile
exit
fi
# scp is part of the openssh package
export SCP=/usr/bin/scp
# Make sure the procps package has been installed
if [ -e /usr/bin/oldps ]; then
PS=/usr/bin/oldps
elif [ -e /usr/bin/procps ]; then
PS=/usr/bin/procps
else
echo "ERROR [$LINENO]$func_name> Missing oldps or procps. Install the procps package." | tee -a logfile
exit
fi
# w is in the procps package
W=/usr/bin/w
# Make sure the email package has been installed
if [ -e /usr/bin/email ]; then
MAIL=/usr/bin/email
else
echo "ERROR [$LINENO]$func_name> Missing email. Install the email package." | tee -a logfile
exit
fi
# Make sure the zip package has been installed
if [ -e /usr/bin/zip ]; then
ZIP_PGM=/usr/bin/zip
else
echo "ERROR [$LINENO]$func_name> Missing zip. Install the zip package." | tee -a logfile
exit
fi
# Make sure the inetutils package has been installed
if [ -e /usr/bin/telnet ]; then
TELNET=/usr/bin/telnet
else
echo "ERROR [$LINENO]$func_name> Missing telnet Install the inetutils package." | tee -a logfile
exit
fi
# cygwin uses the Windows native netstat program
NETSTAT="`cygpath -u -W`/system32/netstat"
LS=/usr/bin/ls
;;
*) # Unknown OS
echo "ERROR [$LINENO]$func_name> Unknown OS" | tee -a logfile
exit
;;
esac
# Set the default values of all environment variables here
CRON_JOB=FALSE
DAEMON_MODE=FALSE
WEB_PAGE_UPDATE_RATE=10
DELETE_STATS=FALSE
DELETE_LOGS=FALSE
EMAIL_ADDRESS="none"
KILL_ACTIVE_PID=FALSE
PROGRAM_NAME=""
SERVER_DOMAIN_NAME=""
SERVER_DOMAIN_PORT=""
SSH_CLIENT=""
SSH_SERVER=""
SSH_SERVER_DIR=""
SHOW_STATS=FALSE
ARG_ST=FALSE
ARG_E=FALSE
LDR_BAD_PORT_CNTR=0
LDR_TIMED_OUT_CNTR=0
LDR_TIMED_OUT_EMAIL_SENT=FALSE
LDR_TIMED_OUT_CNTR2=0
LDR_TIMED_OUT_EMAIL_SENT2=FALSE
LDR_BAD_PORT_EMAIL_SENT=FALSE
LDR_SCP_FAILED_CNTR=0
LDR_SCP_FAILED_EMAIL_SENT=FALSE
TNL_TIMED_OUT_CNTR=0
TNL_TIMED_OUT_EMAIL_SENT=FALSE
TNL_TIMED_OUT_CNTR2=0
TNL_TIMED_OUT_EMAIL_SENT2=FALSE
TNL_SCP_FAILED_CNTR=0
TNL_SCP_FAILED_EMAIL_SENT=FALSE
TNL_BAD_PORT_CNTR=0
TNL_BAD_PORT_EMAIL_SENT=FALSE
# Below this line are possible return codes from functions. Successful answers
# return 0. Non successful answers have non zero values.
ACTIVE=0
DEAD=1
YES=0
NO=1
TRUE=0
FALSE=1
GOOD_PORT=0
BAD_PORT=1
SAME=0
UPDATED=1
SCP_FAILED=64
SCP_PASSED=65
FATAL_ERROR=66
TIMED_OUT=67
} # env_setup
#------------------------------------------------------------------------------
# read_ldr_stats: Function that reads statistics from the loader.stats file
function read_ldr_stats () {
if [ $(($D & 0x3)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside read_ldr_stats" >> logfile; fi
if [ ! -e loader.stats ]; then
# If the statistics file doesn't exist then initialize variables with default values
LOADER_PID=66666
CLIENT_HEARTBEAT="YYYY-MMM-DD HH:MM:SS DDD"
SERVER_HEARTBEAT="YYYY-MMM-DD HH:MM:SS DDD"
SERVER_HEARTBEAT_1="Server Heartbeat 1"
SERVER_HEARTBEAT_2="Server Heartbeat 2"
SERVER_HEARTBEAT_3="Server Heartbeat 3"
SERVER_HEARTBEAT_4="Server Heartbeat 4"
LDR_GOOD_PORT_TESTS=0
LDR_BAD_PORT_TESTS=0
LDR_TIMED_OUT_PORT_TESTS=0
LDR_UPDATED_UPDATE_FILE=0
LDR_TIMED_OUT_UPDATE_FILE=0
LDR_SCP_FAILED_UPDATE_FILE=0
LDR_EMAILS_SENT=0
TUNNEL_KILLED=0
WRONG_LDR_PID_EXITS=0
INACTIVE_TNL_PID_RESTARTS=0
LAST_BAD_PORT_TEST="YYYY-MMM-DD HH:MM:SS DDD"
LAST_LDR_EMAIL="YYYY-MMM-DD HH:MM:SS DDD"
LAST_SCP_FAILED_COPY="YYYY-MMM-DD HH:MM:SS DDD"
TUNNEL_LAST_KILLED="YYYY-MMM-DD HH:MM:SS DDD"
LDR_STATS_CLEARED=`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`
SLEEP_TIME=300
# Since loader.stats doesn't exist it needs to be created
write_ldr_stats
else
# Get the correct fields from the loader.stats file
LOADER_PID=`cat loader.stats | awk '/^------ loader \(pid.*/{print $4}'`
CLIENT_HEARTBEAT=`cat loader.stats | awk '/^client heartbeat:.*/{print $3,$4,$5}'`
SERVER_HEARTBEAT=`cat loader.stats | awk '/^server heartbeat:.*/{print $3,$4,$5}'`
SERVER_HEARTBEAT_1=`cat loader.stats | awk '/^server heartbeat 1.*/{print $4,$5,$6}'`
SERVER_HEARTBEAT_2=`cat loader.stats | awk '/^server heartbeat 2.*/{print $4,$5,$6}'`
SERVER_HEARTBEAT_3=`cat loader.stats | awk '/^server heartbeat 3.*/{print $4,$5,$6}'`
SERVER_HEARTBEAT_4=`cat loader.stats | awk '/^server heartbeat 4.*/{print $4,$5,$6}'`
LDR_GOOD_PORT_TESTS=`cat loader.stats | awk '/^Total good port tests.*/{print $5}'`
LDR_BAD_PORT_TESTS=`cat loader.stats | awk '/^Total failed port tests.*/{print $5}'`
LDR_TIMED_OUT_PORT_TESTS=`cat loader.stats | awk '/^Total timed out port tests.*/{print $6}'`
LDR_UPDATED_UPDATE_FILE=`cat loader.stats | awk '/^Good file copies.*/{print $4}'`
LDR_TIMED_OUT_UPDATE_FILE=`cat loader.stats | awk '/^Timed out file copies.*/{print $5}'`
LDR_SCP_FAILED_UPDATE_FILE=`cat loader.stats | awk '/^Failed scp copies.*/{print $4}'`
LDR_EMAILS_SENT=`cat loader.stats | awk '/^Total emails sent.*/{print $4}'`
TUNNEL_KILLED=`cat loader.stats | awk '/^Total times tunnel killed.*/{print $5}'`
WRONG_LDR_PID_EXITS=`cat loader.stats | awk '/^Total wrong pid exits.*/{print $5}'`
INACTIVE_TNL_PID_RESTARTS=`cat loader.stats | awk '/^Total tunnel inactive PIDs.*/{print $5}'`
LAST_BAD_PORT_TEST=`cat loader.stats | awk '/^Last failed port test.*/{print $5,$6,$7}'`
LAST_LDR_EMAIL=`cat loader.stats | awk '/^Last email sent.*/{print $4,$5,$6}'`
LAST_SCP_FAILED_COPY=`cat loader.stats | awk '/^Last SCP failed copy.*/{print $5,$6,$7}'`
TUNNEL_LAST_KILLED=`cat loader.stats | awk '/^tunnel last killed.*/{print $4,$5,$6}'`
LDR_STATS_CLEARED=`cat loader.stats | awk '/^Stats last cleared.*/{print $4,$5,$6}'`
# Only initialize sleep time from file if -st isn't on command line
if [ "$ARG_ST" != "TRUE" ]; then
SLEEP_TIME=`cat loader.stats | awk '/^Sleep time.*/{print $3}'`
fi
# Only initialize email addresses from file if -e isn't on command line
if [ "$ARG_E" != "TRUE" ]; then
EMAIL_ADDRESS=`cat loader.stats | awk '/^email addresses.*/{print $3}'`
fi
fi
} # read_ldr_stats
#------------------------------------------------------------------------------
# write_ldr_stats: Function that writes to the loader.stats file
function write_ldr_stats () {
if [ $(($D & 0x3)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside write_ldr_stats" >> logfile; fi
cat << EOF > loader.stats
------ loader (pid: ${LOADER_PID:=66666} ) ------------ ($SCRIPT_VER $SCRIPT_DATE) -----
Time: `date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`
client: $SSH_CLIENT server: $SSH_SERVER
client heartbeat: $CLIENT_HEARTBEAT
server heartbeat: $SERVER_HEARTBEAT
server heartbeat 1: $SERVER_HEARTBEAT_1
server heartbeat 2: $SERVER_HEARTBEAT_2
server heartbeat 3: $SERVER_HEARTBEAT_3
server heartbeat 4: $SERVER_HEARTBEAT_4
Total good port tests: $LDR_GOOD_PORT_TESTS
Total failed port tests: $LDR_BAD_PORT_TESTS
Total timed out port tests: $LDR_TIMED_OUT_PORT_TESTS
Good file copies: $LDR_UPDATED_UPDATE_FILE
Timed out file copies: $LDR_TIMED_OUT_UPDATE_FILE
Failed scp copies: $LDR_SCP_FAILED_UPDATE_FILE
Total emails sent: $LDR_EMAILS_SENT
Total times tunnel killed: $TUNNEL_KILLED
Total tunnel inactive PIDs: $INACTIVE_TNL_PID_RESTARTS
Total wrong pid exits: $WRONG_LDR_PID_EXITS
Last failed port test: $LAST_BAD_PORT_TEST
Last email sent: $LAST_LDR_EMAIL
Last SCP failed copy: $LAST_SCP_FAILED_COPY
tunnel last killed: $TUNNEL_LAST_KILLED
Stats last cleared: $LDR_STATS_CLEARED
Sleep time: $SLEEP_TIME
email addresses: $EMAIL_ADDRESS
EOF
} # write_ldr_stats
#------------------------------------------------------------------------------
# read_tnl_stats: Function that reads statistics from the tunnel.stats file
function read_tnl_stats () {
if [ $(($D & 0x5)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside read_tnl_stats" >> logfile; fi
if [ ! -e tunnel.stats ]; then
# If the statistics file doesn't exist then initialize variables with default values
TUNNEL_PID=66666
TNL_CLIENT_HEARTBEAT="YYYY-MMM-DD HH:MM:SS DDD"
TNL_SERVER_HEARTBEAT="YYYY-MMM-DD HH:MM:SS DDD"
WRONG_TNL_PID_EXITS=0
TNL_EMAILS_SENT=0
TNL_GOOD_PORT_TESTS=0
TNL_BAD_PORT_TESTS=0
TNL_TIMED_OUT_PORT_TESTS=0
TNL_UPDATED_UPDATE_FILE=0
TNL_TIMED_OUT_UPDATE_FILE=0
TNL_SCP_FAILED_UPDATE_FILE=0
LAST_TNL_EMAIL="YYYY-MMM-DD HH:MM:SS DDD"
TNL_STATS_CLEARED=`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`
SLEEP_TIME=300
# Since tunnel.stats doesn't exist it needs to be created
write_tnl_stats
else
# Get the correct fields from the tunnel.stats file
TUNNEL_PID=`cat tunnel.stats | awk '/^------ tunnel \(pid.*/{print $4}'`
TNL_CLIENT_HEARTBEAT=`cat tunnel.stats | awk '/^client heartbeat.*/{print $3,$4,$5}'`
TNL_SERVER_HEARTBEAT=`cat tunnel.stats | awk '/^server heartbeat.*/{print $3,$4,$5}'`
WRONG_TNL_PID_EXITS=`cat tunnel.stats | awk '/^Total wrong pid exits.*/{print $5}'`
TNL_EMAILS_SENT=`cat tunnel.stats | awk '/^Total emails sent.*/{print $4}'`
TNL_GOOD_PORT_TESTS=`cat tunnel.stats | awk '/^Total good port tests.*/{print $5}'`
TNL_BAD_PORT_TESTS=`cat tunnel.stats | awk '/^Total failed port tests.*/{print $5}'`
TNL_TIMED_OUT_PORT_TESTS=`cat tunnel.stats | awk '/^Total timed out port tests.*/{print $6}'`
TNL_UPDATED_UPDATE_FILE=`cat tunnel.stats | awk '/^Total updated files.*/{print $4}'`
TNL_TIMED_OUT_UPDATE_FILE=`cat tunnel.stats | awk '/^Total timed out file updates.*/{print $6}'`
TNL_SCP_FAILED_UPDATE_FILE=`cat tunnel.stats | awk '/^Total SCP failed copies.*/{print $5}'`
LAST_TNL_EMAIL=`cat tunnel.stats | awk '/^Last email sent.*/{print $4,$5,$6}'`
TNL_STATS_CLEARED=`cat tunnel.stats | awk '/^Stats last cleared.*/{print $4,$5,$6}'`
fi
} # read_tnl_stats
#------------------------------------------------------------------------------
# write_tnl_stats: Function that writes to the tunnel.stats file
function write_tnl_stats () {
if [ $(($D & 0x5)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside write_tnl_stats" >> logfile; fi
cat << EOF > tunnel.stats
------ tunnel (pid: ${TUNNEL_PID:=66666} ) ------------ ($SCRIPT_VER $SCRIPT_DATE) -----
Time: `date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`
client: $SSH_CLIENT server: $SSH_SERVER
client heartbeat: $TNL_CLIENT_HEARTBEAT
server heartbeat: $TNL_SERVER_HEARTBEAT
Total wrong pid exits: $WRONG_TNL_PID_EXITS
Total emails sent: $TNL_EMAILS_SENT
Total good port tests: $TNL_GOOD_PORT_TESTS
Total failed port tests: $TNL_BAD_PORT_TESTS
Total timed out port tests: $TNL_TIMED_OUT_PORT_TESTS
Total updated files: $TNL_UPDATED_UPDATE_FILE
Total timed out file updates: $TNL_TIMED_OUT_UPDATE_FILE
Total SCP failed copies: $TNL_SCP_FAILED_UPDATE_FILE
Last email sent: $LAST_TNL_EMAIL
Stats last cleared: $TNL_STATS_CLEARED
Sleep time: $SLEEP_TIME
email addresses: $EMAIL_ADDRESS
EOF
} # write_tnl_stats
#------------------------------------------------------------------------------
# read_pls_stats: Function that reads statistics from the pulse.stats file
function read_pls_stats () {
if [ $(($D & 0x9)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside read_pls_stats" >> logfile; fi
if [ ! -e pulse.stats ]; then
# If the statistics file doesn't exist then initialize variables with default values
PULSE_PID=66666
SERVER_HEARTBEAT="YYYY-MMM-DD HH:MM:SS DDD"
CLIENT_HEARTBEAT="YYYY-MMM-DD HH:MM:SS DDD"
CLIENT_HEARTBEAT_1="Client Heartbeat 1"
CLIENT_HEARTBEAT_2="Client Heartbeat 2"
CLIENT_HEARTBEAT_3="Client Heartbeat 3"
CLIENT_HEARTBEAT_4="Client Heartbeat 4"
PLS_EMAILS_SENT=0
WRONG_PLS_PID_EXITS=0
HEARTBEAT_FAILED_EXITS=0
LAST_PLS_EXIT="YYYY-MMM-DD HH:MM:SS DDD"
LAST_PLS_EMAIL="YYYY-MMM-DD HH:MM:SS DDD"
PLS_STATS_CLEARED=`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`
# Since pulse.stats doesn't exist it needs to be created
write_pls_stats
else
# Get the correct fields from the pulse.stats file
PULSE_PID=`cat pulse.stats | awk '/^------ pulse \(pid.*/{print $4}'`
SERVER_HEARTBEAT=`cat pulse.stats | awk '/^server heartbeat:.*/{print $3,$4,$5}'`
CLIENT_HEARTBEAT=`cat pulse.stats | awk '/^client heartbeat:.*/{print $3,$4,$5}'`
CLIENT_HEARTBEAT_1=`cat pulse.stats | awk '/^client heartbeat 1.*/{print $4,$5,$6}'`
CLIENT_HEARTBEAT_2=`cat pulse.stats | awk '/^client heartbeat 2.*/{print $4,$5,$6}'`
CLIENT_HEARTBEAT_3=`cat pulse.stats | awk '/^client heartbeat 3.*/{print $4,$5,$6}'`
CLIENT_HEARTBEAT_4=`cat pulse.stats | awk '/^client heartbeat 4.*/{print $4,$5,$6}'`
PLS_EMAILS_SENT=`cat pulse.stats | awk '/^Total emails sent.*/{print $4}'`
WRONG_PLS_PID_EXITS=`cat pulse.stats | awk '/^Total wrong pid exits.*/{print $5}'`
HEARTBEAT_FAILED_EXITS=`cat pulse.stats | awk '/^Client heartbeat failed exits.*/{print $5}'`
LAST_PLS_EXIT=`cat pulse.stats | awk '/^Last pulse exit.*/{print $4,$5,$6}'`
LAST_PLS_EMAIL=`cat pulse.stats | awk '/^Last email sent.*/{print $4,$5,$6}'`
PLS_STATS_CLEARED=`cat pulse.stats | awk '/^Stats last cleared.*/{print $4,$5,$6}'`
fi
} # read_pls_stats
#------------------------------------------------------------------------------
# write_pls_stats: Function that writes to the pulse.stats file
function write_pls_stats () {
if [ $(($D & 0x9)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside write_pls_stats" >> logfile; fi
cat << EOF > pulse.stats
------ pulse (pid: ${PULSE_PID:=66666} ) ------------ ($SCRIPT_VER $SCRIPT_DATE) -----
Time: `date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`
client: $SSH_CLIENT server: $SSH_SERVER
server heartbeat: $SERVER_HEARTBEAT
client heartbeat: $CLIENT_HEARTBEAT
client heartbeat 1: $CLIENT_HEARTBEAT_1
client heartbeat 2: $CLIENT_HEARTBEAT_2
client heartbeat 3: $CLIENT_HEARTBEAT_3
client heartbeat 4: $CLIENT_HEARTBEAT_4
Total emails sent: $PLS_EMAILS_SENT
Total wrong pid exits: $WRONG_PLS_PID_EXITS
Client heartbeat failed exits: $HEARTBEAT_FAILED_EXITS
Last pulse exit: $LAST_PLS_EXIT
Last email sent: $LAST_PLS_EMAIL
Stats last cleared: $PLS_STATS_CLEARED
Sleep time: $SLEEP_TIME
email addresses: $EMAIL_ADDRESS
EOF
} # write_pls_stats
#------------------------------------------------------------------------------
# kill_ssh_link
# This function searches for any active instances of an ssh link and kills them.
# Set $1 to any non null value to skip the waiting period
function kill_ssh_link () {
if [ $(($D & 0x200005)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside kill_ssh_link" >> logfile; fi
# Isolate pids of active tunnels into array find_pid. First grep for lines that have $SSH_SERVER in them
# and pipe through grep again so that the grep pid doesn't get detected by mistake. Finally pipe the
# remaining lines through awk using the script in $awk_script to isolate the pids into an array.
local awk_script="/.*-F.*$SSH_SERVER/{print \$2}"
local -a find_pid=(`$PS -ef | grep $SSH_SERVER | grep -v grep | awk "$awk_script"`)
if [ $(($D & 0x400000)) -ne 0 ]; then
echo "[$LINENO]$PROGRAM_NAME:$$> kill_ssh_link: Output of ps -ef:" >> logfile
$PS -ef >> logfile
echo "[$LINENO]$PROGRAM_NAME:$$> kill_ssh_link: Contents of find_pid array:" >> logfile
echo ${find_pid[*]} >> logfile
fi
# Kill all the pids
local count=0
while [ "$count" -lt "${#find_pid[*]}" ]; do
kill -9 ${find_pid[$count]} &>/dev/null
log_event "[$LINENO]$PROGRAM_NAME:$$> kill_ssh_link: kill -9 ${find_pid[$count]}"
let count=count+1
done
# Wait awhile to give time for all the ssh junk to be cleaned up by the OS. Attempting to reestablish the tunnel too
# quickly can cause trouble with ports not being forwarded because they are already in use. The tunnel gets set up okay,
# but the ports don't work. This requires setting the tunnel state to disable, letting the tunnel come down, then
# setting state to enable again. That always seems to work. So, hopefully adding a delay here will accomplish the same
# thing. This delay must be less than the daemon cycle time or the cron cycle time. Otherwise the loader heartbeat
# will fail.
if [ "$1" == "" ]; then
sleep 30
fi
} # kill_ssh_link
#------------------------------------------------------------------------------
# get_pulse_pid: Function that returns the pid for "pulse" in RC_GET_PULSE_PID
function get_pulse_pid () {
if [ $(($D & 0x9)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside get_pulse_pid" >> logfile; fi
if [ -e pulse.stats ]; then
RC_GET_PULSE_PID=`cat pulse.stats | awk '/^------ pulse \(pid.*/{print $4}'`
else
RC_GET_PULSE_PID=66666
fi
} # get_pulse_pid
#------------------------------------------------------------------------------
# get_tunnel_pid: Function that returns the pid for "tunnel" in RC_GET_TUNNEL_PID
function get_tunnel_pid () {
if [ $(($D & 0x5)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside get_tunnel_pid" >> logfile; fi
if [ -e tunnel.pid ]; then
RC_GET_TUNNEL_PID=`cat tunnel.pid`
else
RC_GET_TUNNEL_PID=66666
fi
} # get_tunnel_pid
#------------------------------------------------------------------------------
# get_loader_pid: Function that returns the pid for "loader" in RC_GET_LOADER_PID
function get_loader_pid () {
if [ $(($D & 0x3)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside get_loader_pid" >> logfile; fi
if [ -e loader.pid ]; then
RC_GET_LOADER_PID=`cat loader.pid`
else
RC_GET_LOADER_PID=66666
fi
} # get_loader_pid
#------------------------------------------------------------------------------
# fix_number: Function that removes leading zero from seconds and minutes
# returned by `date +%M` and `date +%S`
# $1 - Number to fix
function fix_number () {
local func_name=fix_number
if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> Inside fix_number (number: $1)" >> logfile; fi
case $1 in
00) return 0;;
01) return 1;;
02) return 2;;
03) return 3;;
04) return 4;;
05) return 5;;
06) return 6;;
07) return 7;;
08) return 8;;
09) return 9;;
*) return $1;;
esac
} # fix_number
#------------------------------------------------------------------------------
# logfile: Function that maintains logfiles
function logfile () {
if [ $(($D & 0xD)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside logfile" >> logfile; fi
# Compress logfile once a week and save up to four compressed logs
CURRENT_DOW=`date +%a`
CURRENT_HOUR=`date +%H`
if [ "$CURRENT_DOW" = "Sun" ] && [ "$CURRENT_HOUR" = "00" ] && [ -e logfile ]; then
# Cygwin doesn't gracefully move from one logfile to another and can end up in a condition
# where a new logfile can't be written to. This results in a cron email each time indicating
# "permission denied" when writing to logfile. Rather than put in special code only for
# cygwin, treat all OSs the same. We'll kill the tunnel and any ssh_link processes before
# zipping up the logfile. I think what happens is that the tunnel program tries adding to
# the logfile at the same time the loader is trying to delete it. I noticed that the old
# logfile wasn't visible to cygwin but still showed up in a Windows Explorer. It couldn't
# be deleted. Windows said it was in use. Killing the tunnel and links will stop those
# processes from trying to write to the logfile.
if [ -e logfile.1.zip ]; then
LOGFILE_MONTH=`$LS -n logfile.1.zip | awk '/.*/{print $6}'`
LOGFILE_DAY=`$LS -n logfile.1.zip | awk '/.*/{print $7}'`
CURRENT_MONTH=`date +%b`
CURRENT_DAY=`date +%d`
fix_number $CURRENT_DAY
CURRENT_DAY=$?
if [ "$CURRENT_DAY" != "$LOGFILE_DAY" ] || [ "$CURRENT_MONTH" != "$LOGFILE_MONTH" ]; then
log_event "[$LINENO]$PROGRAM_NAME:$$> logfile: Rotating logfiles. Killing tunnel $RC_GET_TUNNEL_PID and ssh_link."
if [ -e logfile.3.zip ]; then
mv logfile.3.zip logfile.4.zip
fi
if [ -e logfile.2.zip ]; then
mv logfile.2.zip logfile.3.zip
fi
if [ -e logfile.1.zip ]; then
mv logfile.1.zip logfile.2.zip
fi
if [ -e logfile ]; then
# Kill the tunnel and the ssh link
get_tunnel_pid
log_ldr_event "[$LINENO]$PROGRAM_NAME:$$> logfile: Rotating logfiles. Killing tunnel $RC_GET_TUNNEL_PID and ssh_link."
kill -9 $RC_GET_TUNNEL_PID &>/dev/null
kill_ssh_link
# zip up the old logfile, rename it, and delete the old one
$ZIP_PGM logfile.zip logfile
mv logfile.zip logfile.1.zip
# Need to remove the old logfile or else records just get appended
rm -f logfile
fi
fi
else
# No logfile.1.zip so compress logfile, rename, create new, and exit
if [ -e logfile ]; then
log_event "[$LINENO]$PROGRAM_NAME:$$> logfile: Rotating logfiles. Killing tunnel $RC_GET_TUNNEL_PID and ssh_link."
# Kill the tunnel and the ssh link
get_tunnel_pid
kill -9 $RC_GET_TUNNEL_PID &>/dev/null
kill_ssh_link
# zip up the old logfile, rename it, and delete the old one
$ZIP_PGM logfile.zip logfile
mv logfile.zip logfile.1.zip
# Need to remove the old logfile or else records just get appended
rm -f logfile
fi
fi
fi
} # logfile
#------------------------------------------------------------------------------
# read_loader_conf: Function that initializes environment variables from file loader.conf
# Command line arguments have higher priority so if the variable isn't null
# don't set its value.
function read_loader_conf () {
if [ $(($D & 0x3)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside read_loader_conf" >> logfile; fi
if [ ! -e loader.conf ]; then
echo "Error: $WORKING_DIR/loader.conf file missing"
send_ldr_email "[$LINENO] loader.conf file missing in $WORKING_DIR"
log_ldr_event "[$LINENO] loader.conf file missing in $WORKING_DIR"
exit
else
# Initialize environment variables
export SSH_CLIENT=`cat loader.conf | awk '/^# SSH_CLIENT .*/{print $3}'`
export SSH_SERVER=`cat loader.conf | awk '/^# SSH_SERVER .*/{print $3}'`
SSH_SERVER_DIR=`cat loader.conf | awk '/^# SSH_SERVER_DIR .*/{print $3}'`
SERVER_DOMAIN_NAME=`cat loader.conf | awk '/^HostName .*/{print $2}'`
SERVER_DOMAIN_PORT=`cat loader.conf | awk '/^Port .*/{print $3}'`
# Only initialize email addresses from file if -e isn't on command line
if [ "$ARG_E" != "TRUE" ]; then
EMAIL_ADDRESS=`cat loader.conf | awk '/^# EMAIL_ADDRESS .*/{print $3}'`
fi
fi
} # read_loader_conf
#------------------------------------------------------------------------------
# read_tunnel_conf: Function that initializes environment variables from file tunnel.conf
# Command line arguments have higher priority so if the variable isn't null
# don't set its value.
function read_tunnel_conf () {
if [ $(($D & 0x5)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside read_tunnel_conf" >> logfile; fi
if [ ! -e tunnel.conf ]; then
echo "Error: $WORKING_DIR/tunnel.conf file missing"
send_tnl_email "[$LINENO] tunnel.conf file missing in $WORKING_DIR"
log_tnl_event "[$LINENO] tunnel.conf file missing in $WORKING_DIR"
exit
else
# Initialize environment variables
TUNNEL_STATE=`cat tunnel.conf | awk '/^# TUNNEL_STATE .*/{print $3}'`
EMAIL_THRESHOLD=`cat tunnel.conf | awk '/^# EMAIL_THRESHOLD .*/{print $3}'`
NETSTAT_CLIENT_NAME=`cat tunnel.conf | awk '/^# NETSTAT_CLIENT_NAME .*/{print $3}'`
# Only initialize email addresses from file if -e isn't on command line
if [ "$ARG_E" != "TRUE" ]; then
EMAIL_ADDRESS=`cat tunnel.conf | awk '/^# EMAIL_ADDRESS .*/{print $3}'`
fi
# Only initialize sleep time from file if -st isn't on command line
if [ "$ARG_ST" != "TRUE" ]; then
SLEEP_TIME=`cat tunnel.conf | awk '/^# SLEEP_TIME .*/{print $3}'`
fi
fi
} # read_tunnel_conf
#------------------------------------------------------------------
# test_port: Function that tests if a connection can be made to a port on a remote computer
# It returns $GOOD_PORT if able to connect to port and $BAD if connection is refused
# and returns TIMED_OUT if the function times out.
# $1 is the host name
# $2 is the port to test
# $3 is the timeout value. Defaults to 60 if not set.
function test_port () {
if [ $(($D & 0x800007)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside test_port (host: $1 port: $2)" >> logfile; fi
# This local function is what gets timed
function _test_port() {
PORT_TEST="`echo test | $TELNET $1 $2`" 2>1
local result=`echo $PORT_TEST | sed -n -e 's/.* Connected to .*/GOOD_PORT/p' `
if [ $(($D & 0x800000)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> test_port: $PORT_TEST" >> logfile; fi
if [ "$result" = "GOOD_PORT" ]; then
echo $GOOD_PORT >| $$.ret_val
else
echo $BAD_PORT >| $$.ret_val
fi
}
# Initialize the file that gets tested to detect timeout condition
# The timed_function overwrites this with numerical return value when it succeeds
echo "TIMED_OUT" >| $$.ret_val
# Start the timed_function as a subshell in the background
(_test_port $1 $2) &
# Allow up to $3 seconds for timed_function to complete
local count=0
local max_count=${3:-30}
while [ $count -le $max_count ]; do
ret_val=`cat $$.ret_val`
if [ "$ret_val" != "TIMED_OUT" ]; then
rm -f $$.ret_val
RC_TEST_PORT=$ret_val
return $ret_val
fi
let count=count+1
sleep 1
done
# Getting here means timed_function timed out. Kill the subshell $!
kill -9 $! &>/dev/null
log_event "[$LINENO]$PROGRAM_NAME:$$> test_port: Killing _test_port subprocess $!"
rm -f $$.ret_val
RC_TEST_PORT=$TIMED_OUT
return $TIMED_OUT
} # test_port
#------------------------------------------------------------------
# test_port_stats:
# Function that performs statistics on test_port return code.
# $1 is return code
function test_port_stats () {
if [ $(($D & 0x7)) -ne 0 ]; then
echo "[$LINENO]$PROGRAM_NAME:$$> Inside test_port_stats (RC_TEST_PORT: $1 Program: ${PROGRAM_NAME:-not_set})" >> logfile
fi
if [ "${PROGRAM_NAME:=loader}" = "loader" ]; then
case $1 in
$GOOD_PORT) # Good port reported
let LDR_GOOD_PORT_TESTS=LDR_GOOD_PORT_TESTS+1
LDR_TIMED_OUT_CNTR=0
;;
$BAD_PORT) # Port failed connection test
let LDR_BAD_PORT_TESTS=LDR_BAD_PORT_TESTS+1
LAST_BAD_PORT_TEST=`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`
let LDR_BAD_PORT_CNTR=LDR_BAD_PORT_CNTR+1
if [ $LDR_BAD_PORT_CNTR -gt $EMAIL_THRESHOLD ] && [ "$LDR_BAD_PORT_EMAIL_SENT" = "FALSE" ]; then
LDR_BAD_PORT_EMAIL_SENT=TRUE
send_ldr_email "[$LINENO] test_port: bad port occurred"
log_ldr_event "[$LINENO] test_port: bad port occurred"
else
let REMAINDER=LDR_BAD_PORT_CNTR%50
if [ $REMAINDER -eq 0 ]; then
send_ldr_email "[$LINENO] test_port: bad port occurred 50 more times"
log_ldr_event "[$LINENO] test_port: bad port occurred 50 more times"
fi
fi
;;
$TIMED_OUT) # Connection attempt timed out
let LDR_TIMED_OUT_CNTR=LDR_TIMED_OUT_CNTR+1
let LDR_TIMED_OUT_PORT_TESTS=LDR_TIMED_OUT_PORT_TESTS+1
if [ $LDR_TIMED_OUT_CNTR -gt $EMAIL_THRESHOLD ] && [ "$LDR_TIMED_OUT_EMAIL_SENT" = "FALSE" ]; then
LDR_TIMED_OUT_EMAIL_SENT=TRUE
send_ldr_email "[$LINENO] test_port: timeout occurred"
log_ldr_event "[$LINENO] test_port: timeout occurred"
else
let REMAINDER=LDR_TIMED_OUT_CNTR%50
if [ $REMAINDER -eq 0 ]; then
send_ldr_email "[$LINENO] test_port: timeout occurred 50 more times"
log_ldr_event "[$LINENO] test_port: timeout occurred 50 more times"
fi
fi
;;
*) # Unknown test_port return code
echo "ERROR [$LINENO]: Unknown test_port return code: $1 in test_port_stats"
log_ldr_event "ERROR [$LINENO]: Unknown test_port return code: $1 in test_port_stats"
;;
esac
write_ldr_stats
else
# This is tunnel stats processing
case $1 in
$GOOD_PORT) # Good port reported
let TNL_GOOD_PORT_TESTS=TNL_GOOD_PORT_TESTS+1
TNL_TIMED_OUT_CNTR=0
;;
$BAD_PORT) # Port failed connection test
let TNL_BAD_PORT_CNTR=TNL_BAD_PORT_CNTR+1
let TNL_BAD_PORT_TESTS=TNL_BAD_PORT_TESTS+1
if [ $TNL_BAD_PORT_CNTR -gt $EMAIL_THRESHOLD ] && [ "$TNL_BAD_PORT_EMAIL_SENT" = "FALSE" ]; then
TNL_BAD_PORT_EMAIL_SENT=TRUE
send_tnl_email "[$LINENO] test_port: bad port occurred"
log_tnl_event "[$LINENO] test_port: bad port occurred"
else
let REMAINDER=TNL_BAD_PORT_CNTR%50
if [ $REMAINDER -eq 0 ]; then
send_tnl_email "[$LINENO] test_port: bad port occurred 50 more times"
log_tnl_event "[$LINENO] test_port: bad port occurred 50 more times"
fi
fi
;;
$TIMED_OUT) # Connection attempt timed out
let TNL_TIMED_OUT_CNTR=TNL_TIMED_OUT_CNTR+1
let TNL_TIMED_OUT_PORT_TESTS=TNL_TIMED_OUT_PORT_TESTS+1
if [ $TNL_TIMED_OUT_CNTR -gt $EMAIL_THRESHOLD ] && [ "$TNL_TIMED_OUT_EMAIL_SENT" = "FALSE" ]; then
TNL_TIMED_OUT_EMAIL_SENT=TRUE
send_tnl_email "[$LINENO] test_port: timeout occurred"
log_tnl_event "[$LINENO] test_port: timeout occurred"
else
let REMAINDER=TNL_TIMED_OUT_CNTR%50
if [ $REMAINDER -eq 0 ]; then
send_tnl_email "[$LINENO] test_port: timeout occurred 50 more times"
log_tnl_event "[$LINENO] test_port: timeout occurred 50 more times"
fi
fi
;;
*) # Unknown test_port return code
echo "ERROR [$LINENO]: Unknown test_port return code: $1 in test_port_stats"
log_tnl_event "ERROR [$LINENO]: Unknown test_port return code: $1 in test_port_stats"
;;
esac
write_tnl_stats
fi
} # test_port_stats
#------------------------------------------------------------------------------
# display_env: Function to display the values of environment variables
function display_env () {
if [ $(($D & 0xF)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside display_env" >> logfile; fi
cat << EOF >> logfile
---- CURRENT ENVIRONMENT ------------------------
---- Command line variables ---------------------
KILL_ACTIVE_PID: $KILL_ACTIVE_PID
ARG_ST: $ARG_ST
CRON_JOB: $CRON_JOB
DAEMON_MODE: $DAEMON_MODE
DELETE_STATS: $DELETE_STATS
DELETE_LOGS: $DELETE_LOGS
PROGRAM_NAME: $PROGRAM_NAME
SHOW_STATS: $SHOW_STATS
D: $D
WORKING_DIR: $WORKING_DIR
WEB_PAGE_UPDATE_RATE $WEB_PAGE_UPDATE_RATE
---- tunnel.conf variables ----------------------
EMAIL_ADDRESS: $EMAIL_ADDRESS
EMAIL_THRESHOLD: $EMAIL_THRESHOLD
NETSTAT_CLIENT_NAME: $NETSTAT_CLIENT_NAME
SLEEP_TIME: $SLEEP_TIME
TUNNEL_STATE: $TUNNEL_STATE
---- tunnel.stats variables ---------------------
TUNNEL_PID: $TUNNEL_PID
TNL_CLIENT_HEARTBEAT: $TNL_CLIENT_HEARTBEAT
TNL_SERVER_HEARTBEAT: $TNL_SERVER_HEARTBEAT
GOOD_PORT_TESTS: $TNL_GOOD_PORT_TESTS
BAD_PORT_TESTS: $TNL_BAD_PORT_TESTS
TIMED_OUT_PORT_TESTS: $TNL_TIMED_OUT_PORT_TESTS
WRONG_TNL_PID_EXITS: $WRONG_TNL_PID_EXITS
TNL_UPDATED_UPDATE_FILE: $TNL_UPDATED_UPDATE_FILE
TNL_TIMED_OUT_UPDATE_FILE: $TNL_TIMED_OUT_UPDATE_FILE
TNL_SCP_FAILED_UPDATE_FILE: $TNL_SCP_FAILED_UPDATE_FILE
TNL_EMAILS_SENT: $TNL_EMAILS_SENT
LAST_TNL_EMAIL: $LAST_TNL_EMAIL
TNL_STATS_CLEARED: $TNL_STATS_CLEARED
---- loader.conf variables ----------------------
SSH_CLIENT: $SSH_CLIENT
SSH_SERVER: $SSH_SERVER
SSH_SERVER_DIR: $SSH_SERVER_DIR
SERVER_DOMAIN_NAME: $SERVER_DOMAIN_NAME
SERVER_DOMAIN_PORT: $SERVER_DOMAIN_PORT
---- loader.stats variables ---------------------
LOADER_PID: $LOADER_PID
CLIENT_HEARTBEAT: $CLIENT_HEARTBEAT
SERVER_HEARTBEAT: $SERVER_HEARTBEAT
SERVER_HEARTBEAT_1: $SERVER_HEARTBEAT_1
SERVER_HEARTBEAT_2: $SERVER_HEARTBEAT_2
SERVER_HEARTBEAT_3: $SERVER_HEARTBEAT_3
SERVER_HEARTBEAT_4: $SERVER_HEARTBEAT_4
LDR_GOOD_PORT_TESTS: $LDR_GOOD_PORT_TESTS
LDR_BAD_PORT_TESTS: $LDR_BAD_PORT_TESTS
LDR_TIMED_OUT_PORT_TESTS: $LDR_TIMED_OUT_PORT_TESTS
LDR_UPDATED_UPDATE_FILE: $LDR_UPDATED_UPDATE_FILE
LDR_TIMED_OUT_UPDATE_FILE: $LDR_TIMED_OUT_UPDATE_FILE
LDR_SCP_FAILED_UPDATE_FILE: $LDR_SCP_FAILED_UPDATE_FILE
LDR_EMAILS_SENT: $LDR_EMAILS_SENT
INACTIVE_TNL_PID_RESTARTS: $INACTIVE_TNL_PID_RESTARTS
TUNNEL_KILLED: $TUNNEL_KILLED
LAST_BAD_PORT_TEST: $LAST_BAD_PORT_TEST
LAST_LDR_EMAIL: $LAST_LDR_EMAIL
LAST_SCP_FAILED_COPY: $LAST_SCP_FAILED_COPY
TUNNEL_LAST_KILLED: $TUNNEL_LAST_KILLED
LDR_STATS_CLEARED: $LDR_STATS_CLEARED
-------------------------------------------------
EOF
} # display_env
#------------------------------------------------------------------------------
# test_pid: Function that returns ACTIVE if passed pid is active else DEAD
# Pass PID to test in $1
# It puts first 8 characters of the name of the program associated with the PID in $TEST_PID_PGM
function test_pid () {
if [ $(($D & 0x100001)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside test_pid ($1)" >> logfile; fi
# 040224 jrl - The following approach to isolating the pid may seem less efficient than doing
# it all in one line. I was having intermittent problems with the "set" declaring "no match" and
# bombing out of the script, even though the pid was really there. This approach, writing the ps output
# to a file and then using awk to match everything and print field 5 seems to be reliable all the time.
# 040516 jrl - Seemed to be getting dead pid reported even though it was alive. Put test in a
# while loop to try it three times before declaring it failed. See if it makes a difference.
# 040518 jrl - On Solaris the CMD field returned by "ps -p pid" is only 8 characters wide so it
# truncates the program name. Use ${TEST_PID_PGM:0:8} so that all names returned by this program
# are the same length for testing later on.
# 050317 jrl - Cygwin uses "oldps" which has different options and also has data in different
# fields than Linux and Solaris. This function now performs differently based on OS.
# 050709 jrl - Reworked the script to not need a temp file but use internal variables instead.
# Discovered that a typo made cygwin output differently than other OSs. I left off the dash in
# front of the "-p" and that made the output different. Using "-p" makes all of them the same
# so the OS dependency case statement can be removed.
COUNT=3
while [ $COUNT -ne 0 ]; do
local ps_output="`$PS -p $1`"
# The pid is field 5 of the ps output. Isolate that with awk.
TEST_PID=`echo $ps_output | awk '/.*/{print $5}'`
# The program name is field 8 of the ps output. Isolate that with awk.
TEST_PID_PGM=`echo $ps_output | awk '/.*/{print $8}'`
# Truncate program name to 8 characters which is the Solaris length
TEST_PID_PGM=${TEST_PID_PGM:0:8}
if [ $(($D & 0x100000)) -ne 0 ]; then
echo "[$LINENO]$PROGRAM_NAME:$$> test_pid: Contents of \$ps_output" >> logfile
echo $ps_output >> logfile
fi
if [ ! -z $TEST_PID ] && [ $TEST_PID -eq $1 ]; then
RC_TEST_PID=$ACTIVE
return $ACTIVE
fi
let COUNT=COUNT-1
# Wait a little bit to give the OS some time to clean up. False negatives occur at times.
# I'm hoping this will help.
sleep 2
done
if [ $(($D & 0x2000000)) -ne 0 ]; then
# test_pid is often reporting the pid is dead when it really isn't. Capture info in the logfile
# when this happens. Maybe another method needs to be found. This happens mostly on cygwin.
echo "[$LINENO]$PROGRAM_NAME:$$> test_pid: $1 declared dead. Below is output of ps:" >> logfile
$PS -ef >> logfile
fi
RC_TEST_PID=$DEAD
return $DEAD
} # test_pid
#------------------------------------------------------------------------------
# yes_or_no: Function that uses $1 as a prompt and returns $YES or $NO as an answer
function yes_or_no () {
if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside yes_or_no" >> logfile; fi
while [ true ]
do
read -a response -p "$1 (yes/no): "
case ${response[0]} in
yes) RC_YES_OR_NO=$YES
return $YES ;;
no) RC_YES_OR_NO=$NO
return $NO ;;
*) echo "Invalid choice: $response"; echo;;
esac
done
} # yes_or_no
#------------------------------------------------------------------------------
# process_command_line: Function to process the command line
function process_command_line () {
if [ $((${D:=0} & 0x1)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside process_command_line" >> logfile; fi
#local i=1
while [ $# -ne 0 ]
do
case $1 in
-d) #debug mode
# Command line -d overrides environment variable SSH_TUNNEL_DEBUG_FLAGS if it exists, which overrides debug_flags file if it exists
D=$2
if [ "$D" != "0" ] && [ $(($D & 0x80000000)) -eq 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> process_command_line: Debug flags changed to: $D" >> logfile; fi
if [ $(($D & 0x200000)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> process_command_line: In case -d" >> logfile; fi
shift ;;
-c) #cron job processing
if [ $(($D & 0x200000)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> process_command_line: In case -c" >> logfile; fi
CRON_JOB=TRUE;;
-D) # daemon mode
if [ $(($D & 0x200000)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> process_command_line: In case -D" >> logfile; fi
# Reset CRON_JOB to false. Daemon mode has priority over cron mode
CRON_JOB=FALSE
export DAEMON_MODE=TRUE
if [ "$2" -ge 10 -a "$2" -le 3600 ]; then
DAEMON_SLEEP=$2
else
echo "ERROR [$LINENO]: daemon time $2 out of range" | tee -a logfile
exit
fi
shift;;
-ds) #delete statistics
if [ $(($D & 0x200000)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> process_command_line: In case -ds" >> logfile; fi
DELETE_STATS=TRUE
yes_or_no "Remove loader.stats and tunnel.stats?"
if [ "$?" -eq $NO ]; then exit; fi
log_event "[$LINENO] Removing loader.stats and tunnel.stats"
rm -f loader.stats
rm -f tunnel.stats
exit;;
-dl) #delete logs
if [ $(($D & 0x200000)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> process_command_line: In case -dl" >> logfile; fi
DELETE_LOGS=TRUE
yes_or_no "Remove all logfiles?"
if [ "$?" -eq $NO ]; then exit; fi
rm -fr logfile
rm -fr logfile.*.zip
log_event "[$LINENO] logfiles removed `date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`"
exit;;
-e) #email address
if [ $(($D & 0x200000)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> process_command_line: In case -e" >> logfile; fi
EMAIL_ADDRESS=$2
# This flag gives command line precedence over value in tunnel.conf
ARG_E=TRUE
shift ;;
-h) #show help
if [ $(($D & 0x200000)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> process_command_line: In case -h" >> logfile; fi
display_help
exit;;
-hd) #show debug help
if [ $(($D & 0x200000)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> process_command_line: In case -hd" >> logfile; fi
display_debug_help
exit;;
-hs) #show setup help
if [ $(($D & 0x200000)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> process_command_line: In case -hs" >> logfile; fi
display_setup_help
exit;;
-k) #kill running instance
if [ $(($D & 0x200000)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> process_command_line: In case -k" >> logfile; fi
KILL_ACTIVE_PID=TRUE
get_tunnel_pid
get_loader_pid
yes_or_no "Kill loader:$RC_GET_LOADER_PID and tunnel:$RC_GET_TUNNEL_PID?"
if [ "$?" -eq $NO ]; then exit; fi
log_event "[$LINENO] Killing loader, tunnel and any active ssh link using -k"
echo "Killing tunnel: $RC_GET_TUNNEL_PID"
kill -9 $RC_GET_TUNNEL_PID
echo "Killing loader: $RC_GET_LOADER_PID"
kill -9 $RC_GET_LOADER_PID
log_event "[$LINENO] Killing active ssh link using kill_ssh_link"
read_loader_conf
kill_ssh_link NO_WAIT
exit;;
-md) #make a template ssh_tunnel.init.d file
if [ $(($D & 0x2)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case -md" >> logfile; fi
make_ssh_tunnel_init_d
echo "Created: ssh_tunnel.init.d in `pwd`" | tee -a logfile
echo "Be sure to modify it for your particular situation." | tee -a logfile
echo "As root, copy this file to /etc/init.d and rename to ssh_tunnel." | tee -a logfile
echo "As root install the service with this command:" | tee -a logfile
echo " /sbin/chkconfig --add ssh_tunnel" | tee -a logfile
exit;;
-ml) #make a template loader.conf file
if [ $(($D & 0x2)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case -ml" >> logfile; fi
make_loader_conf
echo "Created: loader.conf in `pwd`" | tee -a logfile
echo "Be sure to modify it for your particular situation" | tee -a logfile
exit;;
-mt) #make a template tunnel.conf file
if [ $(($D & 0x2)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case -mt" >> logfile; fi
make_tunnel_conf
echo "Created: tunnel.conf in `pwd`" | tee -a logfile
echo "Be sure to modify it for your particular situation" | tee -a logfile
exit;;
-mw) #make a webpage.copy script for web page updating
if [ $(($D & 0x2)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case -mw" >> logfile; fi
make_copy_script
echo "Created: webpage.copy in `pwd`" | tee -a logfile
echo "Be sure to modify it for your particular situation" | tee -a logfile
exit;;
-p) #program name processing
if [ $(($D & 0x200000)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> process_command_line: In case -p" >> logfile; fi
PROGRAM_NAME=$2
shift ;;
-st) #sleep time processing
if [ $(($D & 0x200000)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> process_command_line: In case -st" >> logfile; fi
SLEEP_TIME=$2
if [ $SLEEP_TIME -ge 10 -a $SLEEP_TIME -le 3600 ]; then
# This flag gives command line precedence over value in tunnel.conf
ARG_ST=TRUE
else
echo "ERROR [$LINENO]: sleep time $SLEEP_TIME out of range" | tee -a logfile
# Maybe put code to send an email here
fi
shift ;;
-s) #show statistics
if [ $(($D & 0x200000)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> process_command_line: In case -s" >> logfile; fi
SHOW_STATS=TRUE
cat loader.stats
cat tunnel.stats
exit;;
-wr) #web page update rate in minutes
if [ $(($D & 0x2)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case -wr" >> logfile; fi
if [ $2 -lt 0 ]; then
display_help
echo "Error: web page update rate must be positive number" | tee -a logfile
exit
fi
WEB_PAGE_UPDATE_RATE=$2
shift ;;
*) #Unknown option
if [ $(($D & 0x200000)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> process_command_line: In case *" >> logfile; fi
display_help
echo "Error: Unknown option: $1 in process_command_line" | tee -a logfile
exit ;;
esac
# Don't shift command line if processing at last argument
if [ $# -ne 0 ]; then
shift
fi
done
} # process_command_line
#------------------------------------------------------------------
# update_file:
# Function that performs a "cmp" on a remote and local file. If the files
# are the same then nothing is done. If they are different, then the file
# is replaced.
#
# The default direction of the transfer is remote to local but can be reversed.
# Returns $SAME if files are the same, $UPDATED if a transfer occurs,
# "TIMED_OUT" if the remote access times out, or $SCP_FAILED if SCP copy fails.
# Usage:
# file_update remote_file_name [options]
# options:
# -cf filename ssh config file (default: loader.conf)
# -h filename host name in config file (default: loader)
# -lf filename local file (/path/filename) (default: basename of remote file)
# -l:r local to remote file transfer
# -r:l remote to local file transfer (default)
# -to time timeout value in seconds (default: 180 seconds)
function update_file () {
if [ $(($D & 0x80007)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside update_file ($*)" >> logfile; fi
# This local function is what gets timed
function _update_file () {
local source_file=$1
local dest_file=$2
local host_name=$3
local config_file=$4
if [ $(($D & 0x30000)) -ne 0 ]; then
# Enable verbose scp info into the logfile
local verbose="-v"
local redirection="2>logfile"
else
# Turn off verbose scp output
local verbose=""
local redirection="2>/dev/null"
fi
# Create the command using eval and then execute the command. Using eval makes it possible to
# change the value of $redirection and have it actually used.
local command="eval $SCP -p $verbose -F $config_file $source_file $dest_file $redirection"
$command
local rc=$?
# If the scp return code was successful then we're all done
if [ "$rc" -eq 0 ]; then
echo $SCP_PASSED >| $$.ret_val
return $SCP_PASSED
else
echo $SCP_FAILED >| $$.ret_val
return $SCP_FAILED
fi
} # _update_file
# This function does the timing once everything is setup
function time_update_file () {
local source_file=$1
local dest_file=$2
local host_name=$3
local config_file=$4
# Initialize the temp file that gets tested to detect timeout condition
# The timed_function overwrites this with numerical return value when it succeeds
echo "TIMED_OUT" >| $$.ret_val
# Start the timed_function as a subshell in the background
(_update_file $source_file $dest_file $host_name $config_file) &
# Allow up to timeout value (default 180) for function to complete
local count=0
local max_count=${timeout:-180}
while [ $count -le $max_count ]; do
ret_val=`cat $$.ret_val`
if [ "$ret_val" != "TIMED_OUT" ]; then
rm -f $$.ret_val
RC_TIME_UPDATE_FILE=$ret_val
return $ret_val
fi
let count=count+1
# If debug enabled, output a tick for each count, "l" if loader and "t" if tunnel
if [ $(($D & 0x20000)) -ne 0 ]; then
if [[ $PROGRAM_NAME = loader ]]; then
printf "l" >> logfile
else
printf "t" >> logfile
fi
fi
sleep 1
done
# Getting here means timed_function timed out. Kill the subshell $!
log_event "[$LINENO]$PROGRAM_NAME:$$> update_file: Killing _update_file subprocess $!"
kill -9 $! &>/dev/null
rm -f $$.ret_val
RC_TIME_UPDATE_FILE=$TIMED_OUT
return $TIMED_OUT
} # time_update_file
# Process the passed command line
while [ $# -ne 0 ]; do
case $1 in
-cf) #ssh config file
local config_file=$2
shift
;;
-h) #host name in ssh config file
local host_name=$2
shift
;;
-lf) #local file (/path/filename)
local local_file=$2
shift
;;
-l:r) #local to remote file transfer
local remote_to_local=$FALSE
;;
-r:l) #remote to local file transfer
local remote_to_local=$TRUE
;;
-to) #Timeout count
local timeout=$2
shift
;;
*) #Remote file to test (/path/filename)
local remote_file=$1
;;
esac
# Don't shift command line if processing at last argument
if [ $# -ne 0 ]; then
shift
fi
done
# Make sure remote_file was given
if [ ! $remote_file ]; then
echo "[$LINENO]$PROGRAM_NAME:$$> update_file: Error - No remote filename given" | tee -a logfile
RC_UPDATE_FILE=$FATAL_ERROR
return $FATAL_ERROR
fi
# Initialize arguments if not on the command line
local local_file=${local_file:-`basename $remote_file`}
local config_file=${config_file:-loader.conf}
local host_name=${host_name:-loader}
local remote_to_local=${remote_to_local:-$TRUE}
# Configure the transfer based on direction
if [[ $remote_to_local -eq $TRUE ]]; then
# Copy the remote file to scp_temp.$$
local source_file="$host_name:$remote_file"
local dest_file="scp_temp.$$"
else
# Copy the local file to the remote host
local source_file="$local_file"
local dest_file="$host_name:$remote_file"
fi
time_update_file $source_file $dest_file $host_name $config_file
# Check the return status of the scp file transfer attempt and return if it timed out
if [[ $RC_TIME_UPDATE_FILE -eq $TIMED_OUT ]]; then
RC_UPDATE_FILE=$TIMED_OUT
return $TIMED_OUT
fi
# Check the return status of the scp file transfer attempt and return if it failed
if [[ $RC_TIME_UPDATE_FILE -eq $SCP_FAILED ]]; then
RC_UPDATE_FILE=$SCP_FAILED
return $SCP_FAILED
fi
# Getting here means the transfer was successful.
# Local to remote file transfers aren't compared. Simply return $SAME since transfer was good.
if [[ $remote_to_local -eq $FALSE ]]; then
RC_UPDATE_FILE=$SAME
return $SAME
fi
# Getting here means a remote to local transfer occurred. Do a binary compare of the temp
# file with the local file. Return $SAME if they are the same.
cmp $local_file scp_temp.$$ &>/dev/null
if [[ $? -eq 0 ]]; then
rm -f scp_temp.$$
RC_UPDATE_FILE=$SAME
return $SAME
fi
# Getting here means the transfer worked and the files are different. Rename the temp file
# to have the correct local name.
mv scp_temp.$$ $local_file
RC_UPDATE_FILE=$UPDATED
return $UPDATED
} # update_file
#------------------------------------------------------------------
# update_file_stats
# $1 is return code
function update_file_stats () {
if [ $(($D & 0x7)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside update_file_stats (RC_UPDATE_FILE: $1 Program: $PROGRAM_NAME)" >> logfile; fi
if [ "$PROGRAM_NAME" = "loader" ] || [ "$CRON_JOB" = "TRUE" ]; then
# Increment loader status counters
case ${1:-""} in
$TIMED_OUT) # Timed out scp copy attempts
let LDR_TIMED_OUT_CNTR2=LDR_TIMED_OUT_CNTR2+1
let LDR_TIMED_OUT_UPDATE_FILE=LDR_TIMED_OUT_UPDATE_FILE+1
if [ $LDR_TIMED_OUT_CNTR2 -gt $EMAIL_THRESHOLD ] && [ "$LDR_TIMED_OUT_EMAIL_SENT2" = "FALSE" ]; then
LDR_TIMED_OUT_EMAIL_SENT2=TRUE
send_ldr_email "[$LINENO] update_file_stats: timeout occurred"
log_ldr_event "[$LINENO] update_file_stats: timeout occurred"
else
let REMAINDER=LDR_TIMED_OUT_CNTR2%50
if [ $REMAINDER -eq 0 ]; then
send_ldr_email "[$LINENO] update_file_stats: timeout occurred 50 more times"
log_ldr_event "[$LINENO] update_file_stats: timeout occurred 50 more times"
fi
fi
;;
$SAME) # Files are the same. Nothing to do.
# Good test occurred so reset the counters
LDR_TIMED_OUT_CNTR2=0
LDR_SCP_FAILED_CNTR=0
;;
$UPDATED) # Files were different so an update was performed
# Good test occurred so reset the counters
LDR_TIMED_OUT_CNTR2=0
LDR_SCP_FAILED_CNTR=0
let LDR_UPDATED_UPDATE_FILE=LDR_UPDATED_UPDATE_FILE+1
send_ldr_email "[$LINENO] $WORKING_DIR/tunnel.conf file updated"
log_ldr_event "[$LINENO] $WORKING_DIR/tunnel.conf file updated"
;;
$SCP_FAILED) # scp copy failed
LAST_SCP_FAILED_COPY=`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`
let LDR_SCP_FAILED_CNTR=LDR_SCP_FAILED_CNTR+1
let LDR_SCP_FAILED_UPDATE_FILE=LDR_SCP_FAILED_UPDATE_FILE+1
if [ $LDR_SCP_FAILED_CNTR -gt $EMAIL_THRESHOLD ] && [ "$LDR_SCP_FAILED_EMAIL_SENT" = "FALSE" ]; then
LDR_SCP_FAILED_EMAIL_SENT=TRUE
send_ldr_email "[$LINENO] update_file_stats: scp failed occurred"
log_ldr_event "[$LINENO] update_file_stats: scp failed occurred"
else
let REMAINDER=LDR_SCP_FAILED_CNTR%50
if [ $REMAINDER -eq 0 ]; then
send_ldr_email "[$LINENO] update_file_stats: scp failed occurred 50 more times"
log_ldr_event "[$LINENO] update_file_stats: scp failed occurred 50 more times"
fi
fi
;;
*) # This should never happen
log_ldr_event "ERROR [$LINENO]: Unknown case: $1 in update_file_stats"
;;
esac
write_ldr_stats
else
# Increment tunnel status counters
case ${1:-""} in
$TIMED_OUT) # Timed out scp copy attempts
let TNL_TIMED_OUT_CNTR2=TNL_TIMED_OUT_CNTR2+1
let TNL_TIMED_OUT_UPDATE_FILE=TNL_TIMED_OUT_UPDATE_FILE+1
if [ $TNL_TIMED_OUT_CNTR2 -gt $EMAIL_THRESHOLD ] && [ "$TNL_TIMED_OUT_EMAIL_SENT2" = "FALSE" ]; then
TNL_TIMED_OUT_EMAIL_SENT2=TRUE
send_tnl_email "[$LINENO] update_file_stats: timeout occurred"
log_tnl_event "[$LINENO] update_file_stats: timeout occurred"
else
let REMAINDER=TNL_TIMED_OUT_CNTR2%50
if [ $REMAINDER -eq 0 ]; then
send_tnl_email "[$LINENO] update_file_stats: timeout occurred 50 more times"
log_tnl_event "[$LINENO] update_file_stats: timeout occurred 50 more times"
fi
fi
;;
$SAME) # Files are the same. Nothing to do.
# Good test occurred so reset the failure counters
TNL_TIMED_OUT_CNTR2=0
TNL_SCP_FAILED_CNTR=0
;;
$UPDATED) # Files were different so an update was performed
# Good test occurred so reset the failure counters
TNL_TIMED_OUT_CNTR2=0
TNL_SCP_FAILED_CNTR=0
let TNL_UPDATED_UPDATE_FILE=TNL_UPDATED_UPDATE_FILE+1
;;
$SCP_FAILED) # scp copy failed
let TNL_SCP_FAILED_CNTR=TNL_SCP_FAILED_CNTR+1
let TNL_SCP_FAILED_UPDATE_FILE=TNL_SCP_FAILED_UPDATE_FILE+1
if [ $TNL_SCP_FAILED_CNTR -gt $EMAIL_THRESHOLD ] && [ "$TNL_SCP_FAILED_EMAIL_SENT" = "FALSE" ]; then
TNL_SCP_FAILED_EMAIL_SENT=TRUE
send_tnl_email "[$LINENO] update_file_stats: scp failed occurred"
log_tnl_event "[$LINENO] update_file_stats: scp failed occurred"
else
let REMAINDER=TNL_SCP_FAILED_CNTR%50
if [ $REMAINDER -eq 0 ]; then
send_tnl_email "[$LINENO] update_file_stats: scp failed occurred 50 more times"
log_tnl_event "[$LINENO] update_file_stats: scp failed occurred 50 more times"
fi
fi
;;
*) # This should never happen
log_tnl_event "ERROR [$LINENO]: Unknown case: $1 in update_file_stats"
;;
esac
write_tnl_stats
fi
} # update_file_stats
#------------------------------------------------------------------------------
# check_server_heartbeat: This function verifies server heartbeats are changing.
# If the heartbeats aren't changing then "tunnel" is killed and restarted.
function check_server_heartbeat () {
if [ $(($D & 0x43)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside check_server_heartbeat" >> logfile; fi
# Read in the server's heartbeat
SERVER_HEARTBEAT=`cat -s heartbeat.$SSH_SERVER`
if [ $(($D & 0x40)) -ne 0 ]; then
echo "[$LINENO]$PROGRAM_NAME:$$> check_server_heartbeat:" >> logfile
echo " SERVER_HEARTBEAT: \"$SERVER_HEARTBEAT\" size: ${#SERVER_HEARTBEAT}" >> logfile
echo " SERVER_HEARTBEAT_1: \"$SERVER_HEARTBEAT_1\" size: ${#SERVER_HEARTBEAT_1}" >> logfile
echo " SERVER_HEARTBEAT_2: \"$SERVER_HEARTBEAT_2\" size: ${#SERVER_HEARTBEAT_2}" >> logfile
echo " SERVER_HEARTBEAT_3: \"$SERVER_HEARTBEAT_3\" size: ${#SERVER_HEARTBEAT_3}" >> logfile
echo " SERVER_HEARTBEAT_4: \"$SERVER_HEARTBEAT_4\" size: ${#SERVER_HEARTBEAT_4}" >> logfile
fi
# Compare the heartbeats. If all are the same then the link is dead.
if [ "$SERVER_HEARTBEAT" == "$SERVER_HEARTBEAT_1" ] &&
[ "$SERVER_HEARTBEAT" == "$SERVER_HEARTBEAT_2" ] &&
[ "$SERVER_HEARTBEAT" == "$SERVER_HEARTBEAT_3" ] &&
[ "$SERVER_HEARTBEAT" == "$SERVER_HEARTBEAT_4" ]; then
# Getting here means the heartbeat files were all the same. That means the "pulse"
# script isn't running to update the heartbeats or that the link is down and data
# can't be transfered. In either case, kill the tunnel and ssh link.
# 050827 jrl - Used to check if link was still active before trying to kill it. Some conditions with
# cygwin would keep the link up even though it was actually dead. This code was never exectured and
# the tunnel stalled. Removed the test and now just brute force kill the tunnel no matter what.
if [ $(($D & 0x40)) -ne 0 ]; then
echo "[$LINENO]$PROGRAM_NAME:$$> check_server_heartbeat: `date +%Y\-%b\-%d\ %H\:%M\:%S` - $SSH_CLIENT to ${SSH_SERVER}: $SSH_SERVER heartbeat died. Killing tunnel (PID: $RC_GET_TUNNEL_PID)" >> logfile
fi
# Increment the tunnel killed counter
let TUNNEL_KILLED=TUNNEL_KILLED+1
write_ldr_stats
# Send email here
send_ldr_email "[$LINENO] check_server_heartbeat: Server heartbeat dead"
log_ldr_event "[$LINENO] check_server_heartbeat: Server heartbeat dead"
# Save time tunnel was killed. Do this after sending the email so email has the previous time it was killed.
TUNNEL_LAST_KILLED=`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`
write_ldr_stats
# Kill the tunnel and the ssh link
get_tunnel_pid
log_ldr_event "[$LINENO]$PROGRAM_NAME:$$> check_server_heartbeat: Killing tunnel $RC_GET_TUNNEL_PID"
kill -9 $RC_GET_TUNNEL_PID &>/dev/null
kill_ssh_link
else
# Getting here means the heartbeats are different and the link is still up. Rotate heartbeats
SERVER_HEARTBEAT_4=$SERVER_HEARTBEAT_3
SERVER_HEARTBEAT_3=$SERVER_HEARTBEAT_2
SERVER_HEARTBEAT_2=$SERVER_HEARTBEAT_1
SERVER_HEARTBEAT_1=$SERVER_HEARTBEAT
fi
} # check_server_heartbeat
#------------------------------------------------------------------------------
# check_client_heartbeat: This function verifies client heartbeats are changing.
# If the heartbeats aren't changing then "tunnel" is killed and restarted.
function check_client_heartbeat () {
if [ $(($D & 0x4009)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside check_client_heartbeat" >> logfile; fi
# Read in the client's heartbeat
CLIENT_HEARTBEAT=`cat -s heartbeat.$SSH_CLIENT`
if [ $(($D & 0x4000)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> check_client_heartbeat: client heartbeat: $CLIENT_HEARTBEAT" >> logfile; fi
# Compare the heartbeats. If all are the same then the link is dead.
if [ "$CLIENT_HEARTBEAT" == "$CLIENT_HEARTBEAT_1" ] &&
[ "$CLIENT_HEARTBEAT" == "$CLIENT_HEARTBEAT_2" ] &&
[ "$CLIENT_HEARTBEAT" == "$CLIENT_HEARTBEAT_3" ] &&
[ "$CLIENT_HEARTBEAT" == "$CLIENT_HEARTBEAT_4" ]; then
# Getting here means the heartbeat files were all the same. That means the "tunnel"
# script isn't running to update the heartbeats or that the link is down and data
# can't be transfered. In either case, exit "pulse".
# Increment the pulse exited counter and update the time
let HEARTBEAT_FAILED_EXITS=HEARTBEAT_FAILED_EXITS+1
write_pls_stats
# Send email here
send_pls_email "[$LINENO] check_client_heartbeat: client heartbeat dead. Exiting pulse"
log_pls_event "[$LINENO] check_client_heartbeat: client heartbeat dead. Exiting pulse"
# Reset all the heartbeats
CLIENT_HEARTBEAT="YYYY-MMM-DD HH:MM:SS DDD"
CLIENT_HEARTBEAT_1="client Heartbeat 1"
CLIENT_HEARTBEAT_2="client Heartbeat 2"
CLIENT_HEARTBEAT_3="client Heartbeat 3"
CLIENT_HEARTBEAT_4="client Heartbeat 4"
# Update exit time in stats
LAST_PLS_EXIT=`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`
# Update the statistics before exiting
write_pls_stats
# Copy pulse.stats to the logfile
cat pulse.stats >> logfile
exit
else
# Getting here means the heartbeats are different and the link is still up. Rotate heartbeats
CLIENT_HEARTBEAT_4=$CLIENT_HEARTBEAT_3
CLIENT_HEARTBEAT_3=$CLIENT_HEARTBEAT_2
CLIENT_HEARTBEAT_2=$CLIENT_HEARTBEAT_1
CLIENT_HEARTBEAT_1=$CLIENT_HEARTBEAT
fi
# Save all the statistics
write_pls_stats
} # check_client_heartbeat
#------------------------------------------------------------------------------
# log_event: This function logs an event from areas of the script common to all
# individual programs within the script.
# $1 - Event to log
function log_event () {
if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside log_event" >> logfile; fi
cat << EOF >> logfile
------ ssh_tunnel (pid: $$) ------------ ($SCRIPT_VER $SCRIPT_DATE) -----
`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a` - $1
EOF
} # log_event
#------------------------------------------------------------------------------
# log_ldr_event: This function logs events from "loader" specific code.
# $1 - Event to log
function log_ldr_event () {
if [ $(($D & 0x3)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside log_ldr_event" >> logfile; fi
cat << EOF >> logfile
------ loader (pid: $$) ------------ ($SCRIPT_VER $SCRIPT_DATE) -----
`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a` - $1
EOF
} # log_ldr_event
#------------------------------------------------------------------------------
# log_pls_event: This function logs events from "loader" specific code.
# $1 - Event to log
function log_pls_event () {
if [ $(($D & 0x9)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside log_pls_event" >> logfile; fi
cat << EOF >> logfile
------ pulse (pid: $$) ------------ ($SCRIPT_VER $SCRIPT_DATE) -----
`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a` - $1
EOF
} # log_pls_event
#------------------------------------------------------------------------------
# log_tnl_event: This function logs events from "tunnel" specific code.
function log_tnl_event () {
if [ $(($D & 0x5)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside log_tnl_event" >> logfile; fi
cat << EOF >> logfile
------ tunnel (pid: $$) ------------ ($SCRIPT_VER $SCRIPT_DATE) -----
`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a` - $1
EOF
} # log_tnl_event
#------------------------------------------------------------------------------
# sleep_time: Function that sleeps the amount passed in $1. The function
# verifies that the sleep time has elapsed. If for some reason the sleep
# ended early, it calculates the remaining time and sleeps again, repeating
# this until the requested sleep period has elapsed. sleep in Cygwin sometimes
# wakes up early causing problems. This gets around that.
function sleep_time () {
if [ $(($D & 0x1000001)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside sleep_time: `date` \$1: $1" >> logfile; fi
# Save the sleep time to a local variable
local sleep_time=$1
# Capture the current time as the number of seconds since 1980 or 1970 depending on OS
CURRENT_TIME=`date +%s`
# Add the sleep time to it to know wakeup time
let WAKEUP_TIME=CURRENT_TIME+sleep_time
sleep $sleep_time
# Now check if the time actually elapsed or not
CURRENT_TIME=`date +%s`
if [ $CURRENT_TIME -lt $WAKEUP_TIME ]; then
local count=0
# Getting here means "sleep" ended early. Sleep the difference.
# Stay in this while loop until the time has elapsed
while [ $CURRENT_TIME -lt $WAKEUP_TIME ]; do
let sleep_time=WAKEUP_TIME-CURRENT_TIME
sleep $sleep_time
CURRENT_TIME=`date +%s`
let count=count+1
# Output debug data if enabled
if [ $(($D & 0x1000000)) -ne 0 ]; then
echo "----------------------------------------------------------" >> logfile
echo "[$LINENO]$PROGRAM_NAME:$$> sleep_time: sleep woke up early: `date`" >> logfile
echo "WAKEUP_TIME: $WAKEUP_TIME" >> logfile
echo "CURRENT_TIME: $CURRENT_TIME" >> logfile
echo "sleep_time: $sleep_time" >> logfile
echo "count: $count" >> logfile
fi
done
# Log event and send out an email that the event occurred
case $PROGRAM_NAME in
loader)
log_ldr_event "[$LINENO] sleep_time: sleep woke up early $count times"
if [ $(($D & 0x1000000)) -ne 0 ]; then
send_ldr_email "[$LINENO] sleep_time: sleep woke up early $count times"
fi
;;
tunnel)
log_tnl_event "[$LINENO] sleep_time: sleep woke up early $count times"
if [ $(($D & 0x1000000)) -ne 0 ]; then
send_tnl_email "[$LINENO] sleep_time: sleep woke up early $count times"
fi
;;
pulse)
log_pls_event "[$LINENO] sleep_time: sleep woke up early $count times"
if [ $(($D & 0x1000000)) -ne 0 ]; then
send_pls_email "[$LINENO] sleep_time: sleep woke up early $count times"
fi
;;
esac
fi
} # sleep_time
#------------------------------------------------------------------------------
# loader:
function loader () {
if [ $(($D & 0x3)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside loader" >> logfile; fi
log_ldr_event "[$LINENO] loader program starting up"
# Initialize all the environment variables. Order is important.
read_loader_conf
read_ldr_stats
read_tnl_stats
read_tunnel_conf
if [ $(($D & 0x80)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> loader: Checking if active loader already running " >> logfile; fi
# Check if $$ matches get_loader_pid and exit if different
get_loader_pid
if [ "$RC_GET_LOADER_PID" -ne $$ ]; then
# Check if the loader pid is active
test_pid $RC_GET_LOADER_PID
if [ "$RC_TEST_PID" -eq $ACTIVE ] && [ "$TEST_PID_PGM" = "ssh_tunn" ]; then
# Getting here means an active ssh_tunnel is running and somehow another one is trying to start.
log_ldr_event "[$LINENO] loader: Active loader $RC_GET_LOADER_PID detected. Exiting $$."
send_ldr_email "[$LINENO] loader: Active loader detected. Exiting $$"
# Create a file for the active ssh_tunnel to test for. If found the counter is incremented.
touch active_loader_detected
exit
fi
fi
# Save $$ to $LOADER_PID so it gets stored in loader.stats
LOADER_PID=$$
write_ldr_stats
if [ $(($D & 0x80)) -ne 0 ]; then
echo "[$LINENO]$PROGRAM_NAME:$$> loader: Current environment:" >> logfile
display_env
fi
if [ $(($D & 0x80)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> loader: Killing any active tunnel that might be running" >> logfile; fi
# Kill "tunnel" and ssh link if active since a new loader is going to start up
get_tunnel_pid
test_pid $RC_GET_TUNNEL_PID
if [ "$RC_TEST_PID" -eq $ACTIVE ] && [ "$TEST_PID_PGM" = "ssh_tunn" ]; then
log_ldr_event "[$LINENO] loader: Active tunnel $RC_GET_TUNNEL_PID detected. Killing it."
let TUNNEL_KILLED=TUNNEL_KILLED+1
write_ldr_stats
send_ldr_email "[$LINENO] Killed active tunnel: $RC_GET_TUNNEL_PID"
TUNNEL_LAST_KILLED=`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`
write_ldr_stats
kill -9 $RC_GET_TUNNEL_PID 2>/dev/null
kill_ssh_link
fi
# Clean up any old *.ret_val files that might be left hanging around
log_ldr_event "[$LINENO] Removing old *.ret_val files"
rm -f *.ret_val
# Calculate web page update rate based on ping rate and force an update
let WEB_PAGE_UPDATE_RATE=WEB_PAGE_UPDATE_RATE*60
let WEB_PAGE_UPDATE_RATE=WEB_PAGE_UPDATE_RATE/$SLEEP_TIME
let WEB_PAGE_UPDATE_RATE=WEB_PAGE_UPDATE_RATE+1
WEB_PAGE_UPDATE_CNTR=0
web_page
# Save the loader's pid to a file so daemon mode can kill it if needed
log_ldr_event "[$LINENO] Saving $LOADER_PID to loader.pid"
echo $LOADER_PID >| loader.pid
# Stay in this loader while loop forever
log_ldr_event "[$LINENO] Entering loader while loop"
while [ true ]; do
if [ $(($D & 0x90)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> At top of loader while loop `date`" >> logfile; fi
# Check if the debug level has changed
set_debug_level
# Check if this loader's pid is the same as the one stored in loader.pid file.
# If it isn't then somehow another loader got started and need to exit so only
# one is running.
if [ "`cat loader.pid`" != "$$" ]; then
# Getting here means another loader got started somehow. Need to notify and exit.
log_ldr_event "[$LINENO] loader: current pid $$ doesn't equal loader.pid `cat loader.pid`. Exiting $$."
send_ldr_email "[$LINENO] loader: current pid $$ doesn't equal loader.pid `cat loader.pid`. Exiting $$."
# Create a file for the active ssh_tunnel to test for. If found the counter is incremented.
touch active_loader_detected
exit
fi
# Check if tunnel state changed to disable and exit loader if it has.
# This test needed because there have been times when the loader kept
# running even though the tunnel was disabled. It needs to be checked
# each time through the while loop.
read_tunnel_conf
if [ $TUNNEL_STATE = "disabled" ]; then
log_ldr_event "[$LINENO] Tunnel state changed to disabled. Exiting loader."
send_ldr_email "[$LINENO] Tunnel state changed to disabled. Exiting loader."
exit
fi
# Write to the heartbeat file so cron can tell the loader is still alive.
# Cron processing uses this to detect a stalled loader.
echo "RUNNING" >| loader.heartbeat
# Update the client's heartbeat
CLIENT_HEARTBEAT=`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`
echo $CLIENT_HEARTBEAT >| heartbeat.$SSH_CLIENT
# Check the server's heartbeat
check_server_heartbeat
# Check ssh server port and update statistics
log_ldr_event "[$LINENO] Verifying server port is good"
test_port $SERVER_DOMAIN_NAME $SERVER_DOMAIN_PORT &>/dev/null
test_port_stats $RC_TEST_PORT
# Only do this if the port is good. No need to even try if the port is down.
if [ "$RC_TEST_PORT" -eq "$GOOD_PORT" ]; then
# Port is good so check for updated tunnel.conf file
log_ldr_event "[$LINENO] Checking if tunnel.conf file updated on server"
update_file $SSH_SERVER_DIR/tunnel.conf
update_file_stats "$RC_UPDATE_FILE"
# Kill the "tunnel" if an updated tunnel.conf was received, reload tunnel.conf, and kill tunnel.
if [ "$RC_UPDATE_FILE" -eq $UPDATED ]; then
let TUNNEL_KILLED=TUNNEL_KILLED+1
write_ldr_stats
TUNNEL_LAST_KILLED=`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`
write_ldr_stats
get_tunnel_pid
log_ldr_event "[$LINENO] Killing tunnel ($RC_GET_TUNNEL_PID) since new tunnel.conf file copied over"
send_ldr_email "[$LINENO] Killing tunnel ($RC_GET_TUNNEL_PID) since new tunnel.conf file copied over"
kill -9 $RC_GET_TUNNEL_PID &>/dev/null
kill_ssh_link
# Read tunnel.conf since an updated version was just received. Need to set the ARG_?
# flags FALSE in case they are leftover true from startup so that tunnel.conf values
# will be read and used.
ARG_E=FALSE
ARG_ST=FALSE
read_tunnel_conf
write_ldr_stats
# Recalculate the web page update rate and force an update since a new tunnel.conf file was copied over
let WEB_PAGE_UPDATE_RATE=WEB_PAGE_UPDATE_RATE*60
let WEB_PAGE_UPDATE_RATE=WEB_PAGE_UPDATE_RATE/$SLEEP_TIME
let WEB_PAGE_UPDATE_RATE=WEB_PAGE_UPDATE_RATE+1
WEB_PAGE_UPDATE_CNTR=0
web_page
# Check if tunnel state changed to disable and exit loader if it has
if [ "$TUNNEL_STATE" = "disabled" ]; then
log_ldr_event "[$LINENO] Tunnel state changed to disabled. Exiting loader."
send_ldr_email "[$LINENO] Tunnel state changed to disabled. Exiting loader."
exit
fi
fi
# Start the "tunnel" if its pid is no longer active. This is the only place the tunnel is started.
get_tunnel_pid
test_pid $RC_GET_TUNNEL_PID
if [ "$RC_TEST_PID" -ne $ACTIVE ]; then
let INACTIVE_TNL_PID_RESTARTS=INACTIVE_TNL_PID_RESTARTS+1
log_ldr_event "[$LINENO] tunnel pid: $RC_GET_TUNNEL_PID inactive. Launching new tunnel."
send_ldr_email "[$LINENO] Tunnel PID $RC_GET_TUNNEL_PID inactive. Launching new tunnel."
$WORKING_DIR/ssh_tunnel -d $D -p tunnel -st $SLEEP_TIME -e $EMAIL_ADDRESS &
# Reset heartbeat values
SERVER_HEARTBEAT="YYYY-MMM-DD HH:MM:SS DDD"
SERVER_HEARTBEAT_1="Server Heartbeat 1"
SERVER_HEARTBEAT_2="Server Heartbeat 2"
SERVER_HEARTBEAT_3="Server Heartbeat 3"
SERVER_HEARTBEAT_4="Server Heartbeat 4"
write_ldr_stats
fi
fi
# Check if another loader started up and increment counters if it did.
if [ -e active_loader_detected ]; then
rm -f active_loader_detected
let WRONG_LDR_PID_EXITS=WRONG_LDR_PID_EXITS+1
let LDR_EMAILS_SENT=LDR_EMAILS_SENT+1
write_ldr_stats
fi
# Copy loader.stats to the logfile
cat loader.stats >> logfile
if [ $(($D & 0x20)) -ne 0 ]; then
echo "[$LINENO]$PROGRAM_NAME:$$> loader: Current stats:" >> logfile
cat loader.stats >> logfile
fi
# Check if time to create and transfer a new web page
web_page
# Save all the stats to file
write_ldr_stats
# Check if the logfiles need to be rotated
logfile
# Sleep awhile and then do it all over again
sleep_time $SLEEP_TIME
done
} # loader
#------------------------------------------------------------------------------
# tunnel:
function tunnel () {
if [ $(($D & 0x805)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside tunnel" >> logfile; fi
log_tnl_event "[$LINENO] tunnel program starting up"
# Initialize all the environment variables. Order is important.
read_loader_conf
read_ldr_stats
read_tnl_stats
read_tunnel_conf
# Save $$ to $TUNNEL_PID so it gets stored in tunnel.stats
TUNNEL_PID=$$
write_tnl_stats
# Save the tunnel's pid to a file so daemon mode can kill it if needed
echo $TUNNEL_PID >| tunnel.pid
log_tnl_event "[$LINENO]$PROGRAM_NAME:$$> tunnel: Checking ssh_server port ($SERVER_DOMAIN_NAME $SERVER_DOMAIN_PORT)"
test_port $SERVER_DOMAIN_NAME $SERVER_DOMAIN_PORT &>/dev/null
test_port_stats $RC_TEST_PORT
while [ $RC_TEST_PORT -ne $GOOD_PORT ]; do
# Stay in here forever until the port checks out
# Check ssh server port and update statistics
test_port $SERVER_DOMAIN_NAME $SERVER_DOMAIN_PORT &>/dev/null
test_port_stats $RC_TEST_PORT
log_tnl_event "[$LINENO] Waiting for server port ($SERVER_DOMAIN_NAME $SERVER_DOMAIN_PORT) to check good."
sleep_time $SLEEP_TIME
done
# Getting here means the server's ssh port is good.
# Copy the ssh_tunnel file to the ssh server
while [ true ]; do
log_tnl_event "[$LINENO] Copying ssh_tunnel script to server"
update_file $SSH_SERVER_DIR/ssh_tunnel -l:r
update_file_stats $RC_UPDATE_FILE
if [ "$RC_UPDATE_FILE" -eq $UPDATED ] || [ $RC_UPDATE_FILE -eq $SAME ]; then break; fi
log_tnl_event "[$LINENO] Waiting for copy of ssh_tunnel script to server to complete."
sleep_time $SLEEP_TIME
done
# Copy the loader.conf file to the ssh server
while [ true ]; do
log_tnl_event "[$LINENO] Copying loader.conf file to server"
update_file $SSH_SERVER_DIR/loader.conf -l:r
update_file_stats $RC_UPDATE_FILE
if [ "$RC_UPDATE_FILE" -eq $UPDATED ] || [ $RC_UPDATE_FILE -eq $SAME ]; then break; fi
log_tnl_event "[$LINENO] Waiting for copy of loader.conf script to server to complete."
sleep_time $SLEEP_TIME
done
# Look for previous ssh connection to ssh server and kill it
log_tnl_event "[$LINENO] Checking for active ssh_links to kill"
kill_ssh_link
# Getting here means it was possible to copy ssh_tunnel and tunnel.conf over to the ssh server.
# Make the ssh connection to the ssh server
log_tnl_event "[$LINENO] Initiating ssh tunnel with server"
if [ $(($D & 0x400)) -ne 0 ]; then
$SSH -v -F `pwd`/tunnel.conf -n ssh_tunnel $SSH_SERVER_DIR/ssh_tunnel -d $D -p pulse -st $SLEEP_TIME -e $EMAIL_ADDRESS 2>&1 >> logfile &
else
$SSH -F `pwd`/tunnel.conf -n ssh_tunnel $SSH_SERVER_DIR/ssh_tunnel -d $D -p pulse -st $SLEEP_TIME -e $EMAIL_ADDRESS 2> /dev/null >> logfile &
fi
if [ $(($D & 0x800)) -ne 0 ]; then
echo "[$LINENO]$PROGRAM_NAME:$$> tunnel: Current environment just before while loop:" >> logfile
display_env
fi
# Stay in this tunnel while loop forever
log_tnl_event "[$LINENO] Entering tunnel while loop"
while [ true ]; do
if [ $(($D & 0x900)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> tunnel: At top of tunnel while loop `date`" >> logfile; fi
# Check if the debug level has changed
set_debug_level
# Check if tunnel state changed to disable and exit tunnel if it has.
# This test needed because there have been times when the tunnel kept
# running even though the tunnel was disabled. It needs to be checked
# each time through the while loop.
read_tunnel_conf
if [ $TUNNEL_STATE = "disabled" ]; then
log_tnl_event "[$LINENO] Tunnel state changed to disabled. Exiting tunnel."
send_tnl_email "[$LINENO] Tunnel state changed to disabled. Exiting tunnel."
exit
fi
# Check if $$ matches $TUNNEL_PID and exit if different
get_tunnel_pid
if [ "$RC_GET_TUNNEL_PID" -ne $$ ]; then
log_tnl_event "[$LINENO] Current PID $$ doesn't equal \$TUNNEL_PID $RC_GET_TUNNEL_PID. Exiting.";
send_tnl_email "[$LINENO] Current PID $$ doesn't equal \$TUNNEL_PID $RC_GET_TUNNEL_PID. Exiting.";
echo `date +%Y\-%b\-%d\ %H\:%M\:%S` >| tunnel_wrong_pid_exit
exit
fi
# This method uses ssh instead of scp. scp seems to hang on XP clients when using the "heartbeat" host.
# Testing has shown that I can still ssh through the tunnel even though scp doesn't work. Trying this
# as a work around.
TNL_SERVER_HEARTBEAT=`$SSH -F tunnel.conf -n heartbeat cat $SSH_SERVER_DIR/heartbeat.$SSH_SERVER` 2>/dev/null
log_tnl_event "[$LINENO] Copied server heartbeat to client: $TNL_SERVER_HEARTBEAT"
echo "$TNL_SERVER_HEARTBEAT" >| heartbeat.$SSH_SERVER
$SSH -F tunnel.conf -n heartbeat "echo `date` >| $SSH_SERVER_DIR/heartbeat.$SSH_CLIENT" 2>/dev/null
TNL_CLIENT_HEARTBEAT=`$SSH -F tunnel.conf -n heartbeat cat $SSH_SERVER_DIR/heartbeat.$SSH_CLIENT` 2>/dev/null
log_tnl_event "[$LINENO] Copied client heartbeat to server: $TNL_CLIENT_HEARTBEAT"
echo "$TNL_CLIENT_HEARTBEAT" >| heartbeat.$SSH_CLIENT
# Check if another tunnel exited due to a wrong pid detected
if [ -e tunnel_wrong_pid_exit ]; then
let WRONG_TNL_PID_EXITS=WRONG_TNL_PID_EXITS+1
rm -f tunnel_wrong_pid_exit
let TNL_EMAILS_SENT=TNL_EMAILS_SENT+1
fi
# Save all latest stats to file
write_tnl_stats
# Copy tunnel.stats to the logfile
cat tunnel.stats >> logfile
if [ $(($D & 0x200)) -ne 0 ]; then
echo "[$LINENO]$PROGRAM_NAME:$$> tunnel: Current stats:" >> logfile
cat tunnel.stats >> logfile
fi
sleep_time $SLEEP_TIME
done
} # tunnel
#------------------------------------------------------------------------------
# pulse
function pulse () {
if [ $(($D & 0x8009)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> pulse function starting up" >> logfile; fi
# Send this back to the ssh client
echo "[$LINENO]$PROGRAM_NAME:$$> pulse function starting up"
# Initialize all the environment variables. Order is important.
if [ $(($D & 0x8000)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> pulse: Calling read_loader_conf and read_pls_stats" >> logfile; fi
read_loader_conf
read_tunnel_conf
read_pls_stats
# update the client heartbeat values
CLIENT_HEARTBEAT_1="Client Heartbeat 1"
CLIENT_HEARTBEAT_2="Client Heartbeat 2"
CLIENT_HEARTBEAT_3="Client Heartbeat 3"
CLIENT_HEARTBEAT_4="Client Heartbeat 4"
# Initialize netstat email flag to false
NETSTAT_EMAIL_SENT="FALSE"
# Save $$ to $PULSE_PID for later use
PULSE_PID=$$
write_pls_stats
if [ $(($D & 0x8000)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> pulse: Current environment just before while loop:"; display_env; fi
send_pls_email "[$LINENO] Starting up pulse (pid: $$)";
# Stay in this pulse while loop forever
while [ true ]; do
if [ $(($D & 0x9000)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> At top of pulse while loop `date`"; fi
# Check if the debug level has changed
set_debug_level
# Check if $$ matches $PULSE_PID and exit if different
get_pulse_pid
if [ "$RC_GET_PULSE_PID" -ne $$ ]; then
# Need to use $RC_GET_PULSE_PID as PULSE_PID so that it doesn't get overwritten when using write_pls_stats
PULSE_PID=$RC_GET_PULSE_PID
# Create the file so wrong pid exit can be counted by other pulse process
echo `date +%Y\-%b\-%d\ %H\:%M\:%S` >| pulse_wrong_pid_exit
log_pls_event "[$LINENO] Current PID $$ doesn't equal \$PULSE_PID $RC_GET_PULSE_PID. Exiting."
# Send info back to ssh client
echo "[$LINENO] Current PID $$ doesn't equal \$PULSE_PID $RC_GET_PULSE_PID. Exiting."
exit
fi
# Update the server's heartbeat
SERVER_HEARTBEAT=`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`
echo $SERVER_HEARTBEAT >| heartbeat.$SSH_SERVER
# Code to check heartbeats here
check_client_heartbeat
# Check if another pulse exited due to a wrong pid detected
if [ -e pulse_wrong_pid_exit ]; then
let WRONG_PLS_PID_EXITS=WRONG_PLS_PID_EXITS+1
LAST_PLS_EXIT=`cat -s pulse_wrong_pid_exit`
send_pls_email "[$LINENO] Wrong pid exit occurred at $LAST_PLS_EXIT";
rm -f pulse_wrong_pid_exit
fi
# Check if client IP has changed, but only if NETSTAT_CLIENT_NAME isn't blank in tunnel.conf
if [ "$NETSTAT_CLIENT_NAME" != "" ]; then
COUNT=0
# Check three times if the client IP name is missing. Sometimes the OS fails to setup the
# pipe giving a false null. If the test is null three times then confidence is high that
# the IP has changed.
while [ $COUNT -lt 3 ]; do
NETSTAT_NAME_FOUND=`$NETSTAT | grep $NETSTAT_CLIENT_NAME`
if [ "$NETSTAT_NAME_FOUND" == "" ]; then
let COUNT=COUNT+1
else
# Getting here means the IP name was found. Break out of the while loop.
break
fi
done
# Now check if the null count got up to 3. If so, send an email if one hasn't already been sent.
if [ "$COUNT" -ge 3 ]; then
if [ "$NETSTAT_EMAIL_SENT" = "FALSE" ]; then
# Client IP address has changed from what is in /etc/hosts. Send an email once.
NETSTAT_EMAIL_SENT="TRUE"
send_netstat_email
fi
else
# Reset the email sent flag. If the /etc/hosts file is updated with the new IP address
# after the email is received, then netstat will start finding the host name again. Clearing
# the flag here will allow a future IP change to be detected without having to restart the
# tunnel.
NETSTAT_EMAIL_SENT="FALSE"
fi
fi
# Save current statistics
write_pls_stats
# Copy pulse.stats to the logfile
cat pulse.stats >> logfile
# Echo pulse stats back to client
cat pulse.stats
# Check if logfiles need to be rotated
logfile
sleep_time $SLEEP_TIME
done
} # pulse
#------------------------------------------------------------------------------
# send_ldr_email
# $1 - The email subject
function send_ldr_email () {
if [ $(($D & 0x3)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside send_ldr_email: $EMAIL_ADDRESS" >> logfile; fi
if [ $EMAIL_ADDRESS == "none" ]; then return; fi
let LDR_EMAILS_SENT=LDR_EMAILS_SENT+1
write_ldr_stats
local email_time=`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`
echo "$email_time - loader" >| $$.email
echo "$SSH_CLIENT to ${SSH_SERVER}: $1" >> $$.email
cat loader.stats >> $$.email
cat tunnel.stats >> $$.email
cat $$.email | $MAIL -s "ssh: $SSH_CLIENT to ${SSH_SERVER}: $1" $EMAIL_ADDRESS
rm -f $$.email
LAST_LDR_EMAIL=$email_time
write_ldr_stats
} # send_ldr_email
#------------------------------------------------------------------------------
# send_pls_email
# $1 - The email subject
function send_pls_email () {
if [ $(($D & 0x9)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside send_pls_email: $EMAIL_ADDRESS" >> logfile; fi
if [ $EMAIL_ADDRESS == "none" ]; then return; fi
let PLS_EMAILS_SENT=PLS_EMAILS_SENT+1
write_pls_stats
local email_time=`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`
echo "$email_time - pulse" >| $$.email
echo "$SSH_CLIENT to ${SSH_SERVER}: $1" >> $$.email
cat pulse.stats >> $$.email
cat $$.email | $MAIL -s "ssh: $SSH_CLIENT to ${SSH_SERVER}: $1" $EMAIL_ADDRESS
rm -f $$.email
LAST_PLS_EMAIL=$email_time
write_pls_stats
} # send_pls_email
#------------------------------------------------------------------------------
# send_netstat_email: Function used to make the email sent when the client's IP address changes.
function send_netstat_email () {
if [ $(($D & 0x9)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside send_netstat_email: $EMAIL_ADDRESS" >> logfile; fi
if [ $EMAIL_ADDRESS == "none" ]; then return; fi
let PLS_EMAILS_SENT=PLS_EMAILS_SENT+1
write_pls_stats
local email_time=`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`
echo "$email_time - pulse" >| $$.email
echo "$SSH_CLIENT to ${SSH_SERVER}: $1" >> $$.email
echo "Client's IP address has changed. It no longer matches what is in hosts file." >> $$.email
echo "Using NETSTAT_CLIENT_NAME: $NETSTAT_CLIENT_NAME" >> $$.email
echo "" >> $$.email
echo "Below is contents of /etc/hosts file" >> $$.email
echo "" >> $$.email
cat /etc/hosts >> $$.email
echo "" >> $$.email
echo "Below is output of netstat showing domain names" >> $$.email
echo "" >> $$.email
$NETSTAT >> $$.email
echo "" >> $$.email
echo "Below is output of netstat showing IP addresses" >> $$.email
echo "" >> $$.email
$NETSTAT -n >> $$.email
cat $$.email | $MAIL -s "ssh: $SSH_CLIENT IP Changed!!" $EMAIL_ADDRESS
rm -f $$.email
LAST_PLS_EMAIL=$email_time
write_pls_stats
} # send_netstat_email
#------------------------------------------------------------------------------
# send_tnl_email
# $1 - The email subject
function send_tnl_email () {
if [ $(($D & 0x5)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside send_tnl_email: $EMAIL_ADDRESS" >> logfile; fi
if [ $EMAIL_ADDRESS == "none" ]; then return; fi
let TNL_EMAILS_SENT=TNL_EMAILS_SENT+1
write_tnl_stats
local email_time=`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`
echo "$email_time - tunnel" >| $$.email
echo "$SSH_CLIENT to ${SSH_SERVER}: $1" >> $$.email
cat tunnel.stats >> $$.email
cat loader.stats >> $$.email
cat $$.email | $MAIL -s "ssh: $SSH_CLIENT to ${SSH_SERVER}: $1" $EMAIL_ADDRESS
rm -f $$.email
LAST_TNL_EMAIL=$email_time
write_tnl_stats
} # send_tnl_email
#------------------------------------------------------------------------------
# make_copy_script: Function to create a template webpage.copy script
function make_copy_script () {
local func_name=make_copy_script
if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> Inside make_copy_script" >> logfile; fi
cat << EOF >| webpage.copy
#!/bin/bash
#----- ssh_tunnel ($SCRIPT_VER $SCRIPT_DATE) --------------------------------------------
# IMPORTANT: YOU MUST MODIFY THIS FILE FOR YOUR SITUATION.
# This is the webpage.copy script called by ssh_tunnel to update the website. The name of the
# html file is passed in \$1 so that it can be renamed as needed. This might be
# required if multiple sites are testing the same destination IP and all the
# web pages are in the same directory on the webserver.
#
# The following environment variables are exported by ssh_tunnel and can be used in this script:
# SSH_CLIENT Name of the ssh client
# SSH_SERVER Name of the ssh server
# SCP The full path to the scp program on the host where the test is running
#
# The copying method will vary depending on how to access the web server. If it
# is on the same LAN as the machine running the test, then a simple "cp" command
# might work. If the webserver is on a remote machine then it may be necessary
# to use "scp" to do the copy using already established public/private RSA key
# access. Examples of both are given below.
# LAN copy. Uncomment the following line to do a simple LAN copy to the webserver.
#cp -f \$1 /path/to/webserver/directory/\${SSH_CLIENT}_to_\$SSH_SERVER.html
# SCP copy. Use this method if transferring the file to a remote webserver. The "-F" option
# tells scp to use the named ssh configuration file. Below are the settings that might
# be in such a file. See manpage for ssh_config(5) for complete descriptions:
#
# Host loader
# HostName linux-beast
# Port = 2222
# UserKnownHostsFile = /home/jlarsen/ssh_tunnel/linux-beast/known_hosts
# User = jlarsen
# IdentityFile = /home/jlarsen/ssh_tunnel/linux-beast/id_rsa.fhc-beast
#
# Using the settings above and the private key (with no pass phrase) the transfer can be
# done by a cron job.
#
function copy_html () {
# It is important that the file copy operation be performed in such a way that it can be terminated
# if something fails during the scp transfer. The following function is called in a subshell. Before
# the function is called, a temporary file webcopy.ret_val is initialized to "TIMED_OUT". If the
# transfer is successful, the function overwrites this with "SUCCESS". The value of webcopy.ret_val
# is tested once per second in a while loop. The loop exits when webcopy.ret_val becomes "SUCCESS".
# If the scp transfer fails, the while loop times out and the subshell is killed.
#
# This is the function that gets timed
function _copy_html () {
echo "TIMED_OUT" >| webcopy.ret_val
# !!! MODIFY THE FOLLOWING LINE WITH CORRECT SSH CONFIG FILE AND THE FILE LOCATION ON WEBSERVER !!!
\$SCP -F /path/to/ssh/config/file \$1 loader:/path/to/webserver/directory/\${SSH_CLIENT}_to_\$SSH_SERVER.html 2>/dev/null
if [ "\$?" -eq 0 ]; then
# Getting here means the scp copy was successful
echo SUCCESS >| webcopy.ret_val
fi
} # _copy_html
# Start the timed_function as a subshell in the background
(_copy_html \$1) &
# Allow up to 180 seconds for the copy to complete
count=0
while [ \$count -le 180 ]; do
if [ "\`cat webcopy.ret_val\`" == "SUCCESS" ]; then
# Getting here means the scp copy was successful
rm -f webcopy.ret_val
exit
fi
let count=count+1
sleep 1
done
# Getting here means copy_html timed out. Kill the subshell \$!
kill -9 \$! &>/dev/null
rm -f webcopy.ret_val
} # copy_html
# Uncomment the following line to use scp to copy the webpage to the webserver
#copy_html \$1
#Copyright: 2004 and 2005 by John R Larsen
#Free for personal use. Contact the author for commercial use.
#http://larsen-family.us theClaw56@larsen-family.us
EOF
# Set execute permissions on the webpage.copy script
chmod 755 webpage.copy
} # make_copy_script
#------------------------------------------------------------------------------
# make_ssh_tunnel_init_d: Function to create a template loader.conf file
function make_ssh_tunnel_init_d () {
local func_name=make_ssh_tunnel_init_d
if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> Inside make_ssh_tunnel_init_d" >> logfile; fi
cat << EOF >| ssh_tunnel.init.d
#!/bin/bash
#
# Version: $SCRIPT_VER $SCRIPT_DATE
#
# chkconfig: 2345 95 95
# description: Manages starting and stopping of ssh_tunnel
#
# processname: ssh_tunnel
#
# The daemon sleep time in seconds is set below
daemon_sleep_time=300
# The path to the ssh_tunnel working directory is given below
ssh_tunnel_path="`pwd`"
#------------------------------------------------------------------------------
# test_pid: Function that returns 0 if ssh_tunnel pid is active and program
# name is "ssh_tunnel" else returns 1
function test_pid () {
# Get saved pid from file. If file doesn't exist then use a dummy value.
if [ -e \$ssh_tunnel_path/ssh_tunnel.pid ]; then
ssh_tunnel_pid=\`cat \$ssh_tunnel_path/ssh_tunnel.pid\`
else
ssh_tunnel_pid=1234
fi
# Sometimes the OS doesn't setup the pipes correctly and a false failure occurs.
# Try this in a loop 3 times before declaring a failure.
count=3
while [ \$count -ne 0 ]; do
# Isolating pid and program name this way works reliably on Solaris and Linux
local ps_output="\`\$PS -p \$1\`"
# The pid is field 5 of the ps output. Isolate that with awk.
TEST_PID=\`echo \$ps_output | awk '/.*/{print \$5}'\`
# The program name is field 8 of the ps output. Isolate that with awk.
TEST_PID_PGM=\`echo \$ps_output | awk '/.*/{print \$8}'\`
# Truncate program name to 8 characters which is the Solaris length
TEST_PID_PGM=\${TEST_PID_PGM:0:8}
# Check if active pid number existed
if [ ! -z \$TEST_PID ] && [ \$TEST_PID -eq \$ssh_tunnel_pid ]; then
# pid is active, but is it the ssh_tunnel program?
case \$TEST_PID_PGM in
# Older bash versions don't support the \${VARIABLE:n:n} syntax. Use case statement.
ssh_tunn*)
return 0
esac
fi
let count=count-1
done
return 1
} # test_pid
start()
{
# Check if process is running first. If so, then report pid and exit.
test_pid
if [ "\$?" == "0" ] ; then
echo "ssh_tunnel is already running (pid: \$ssh_tunnel_pid)"
exit 0
fi
# Check if file exists before trying to start
if [ ! -x "\$ssh_tunnel_path/ssh_tunnel" ] ; then
echo "\$ssh_tunnel_path/ssh_tunnel doesn't exist or isn't executable."
exit 1
fi
# Start the ssh_tunnel
echo "\`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a\` - daemon: Starting everything with /etc/init.d/ssh_tunnel start" >> \$ssh_tunnel_path/logfile
# Run process as root and redirect stderr and stdout to /dev/null so console isn't flooded with output
# The number after -D is how many seconds ssh_tunnel sleeps. Change this to the value you want.
su -s /bin/bash - -c "\$ssh_tunnel_path/ssh_tunnel -D \$daemon_sleep_time" &>/dev/null &
sleep 10
# Verify ssh_tunnel is running
test_pid
if [ "\$?" == "0" ] ; then
echo "Started ssh_tunnel (PID: \$ssh_tunnel_pid)"
else
echo "ssh_tunnel failed to start"
exit 1
fi
}
stop()
{
# Stop the ssh_tunnel if it is running else report that it isn't running
test_pid
if [ "\$?" == "0" ] ; then
echo "\`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a\` - daemon: Killing everything with /etc/init.d/ssh_tunnel stop" >> \$ssh_tunnel_path/logfile
kill -9 \$ssh_tunnel_pid
kill -9 \`cat \$ssh_tunnel_path/loader.pid\`
kill -9 \`cat \$ssh_tunnel_path/tunnel.pid\`
sleep 1
test_pid
if [ "\$?" == "0" ] ; then
echo "ssh_tunnel failed to stop"
exit 1
else
echo "Stopped ssh_tunnel"
fi
else
echo "ssh_tunnel not running"
exit 0
fi
}
restart()
{
stop
start
}
status()
{
test_pid
if [ "\$?" == "0" ] ; then
echo "ssh_tunnel IS running"
echo "ssh_tunnel daemon: \$ssh_tunnel_pid"
echo "loader: \`cat \$ssh_tunnel_path/loader.pid\`"
echo "tunnel: \`cat \$ssh_tunnel_path/tunnel.pid\`"
else
echo "ssh_tunnel is NOT running"
exit 1
fi
}
case "\$1" in
start)
start
;;
stop)
stop
;;
status)
status
;;
restart)
restart
;;
*)
echo "Usage: \`basename \$0\` { start | stop | status | restart }"
exit 1
esac
exit \$?
EOF
chmod 755 ssh_tunnel.init.d
} # make_ssh_tunnel_init_d
#------------------------------------------------------------------------------
# make_loader_conf: Function to create a template loader.conf file
function make_loader_conf () {
local func_name=make_loader_conf
if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> Inside make_loader_conf" >> logfile; fi
cat << EOF >| loader.conf
# Filename: loader.conf
# Description: This is the configuration file for the loader portion of the
# ssh_tunnel program. The contents of this file never change. This insures
# that the loader will always operate. It is important not to change the
# formatting of thse file because it is read by the ssh_tunnel and processed
# using awk which is expecting things to be in certain locations.
#
# loader.conf has two sections:
# 1. Configuration information such as the names of the ssh and sshd
# hosts, the working directories on each host, and other data as needed.
# 2. The ssh host configuration for the loader program is in this file.
# That configuration should never change. It needs to be at the end
# of the file because ssh uses everything between "Host" entries as
# configuration. There is only one Host section in this file.
#
########################################################################
# The following section contains host information
# SSH_CLIENT john-beast
# SSH_SERVER linux-beast
# SSH_SERVER_DIR /home/jlarsen/ssh_tunnel/john-beast
# EMAIL_ADDRESS none
#
########################################################################
# The following section is the ssh config file for the loader program.
Host loader
HostName linux-beast
Port = 2222
UserKnownHostsFile = /home/jlarsen/ssh_tunnel/linux-beast/known_hosts
User = jlarsen
IdentityFile = /home/jlarsen/ssh_tunnel/linux-beast/id_rsa.fhc-beast
EOF
} # make_loader_conf
#------------------------------------------------------------------------------
# make_tunnel_conf: Function to create a template tunnel.conf file
function make_tunnel_conf () {
local func_name=make_tunnel_conf
if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> Inside make_tunnel_conf" >> logfile; fi
cat << EOF >| tunnel.conf
# Filename: tunnel.conf
#
# Description: This is the configuration file for the tunnel portion of the
# ssh_tunnel program. The contents of this file can be changed during program execution.
# It is important not to change the formatting of the file because it is read by the
# tunnel program and processed using awk which is expecting things to be in certain
# locations.
#
# tunnel.conf has two sections:
# 1. Configuration information such as email addresses and thresholds.
# 2. The ssh host configuration for the tunnel. This consists of two
# host definitions, tunnel and heartbeat. The tunnel is the permanent
# connection and defines port forwarding. The heartbeat uses one of
# the forwarded ports to write and copy heartbeat files between the
# ssh client and the ssh server. Changing heartbeats indicate that the
# tunnel is functioning.
#
########################################################################
# The following section contains configuration information
#
# The tunnel is either "enabled" or "disabled" based on the value given
# below. If disabled, then the no processes are running on either the
# client or the server. A cron job on the client runs ssh_tunnel periodically
# to check if tunnel.conf has changed on the server. If a change is detected
# then the new tunnel.conf file is transfered over. If the tunnel state
# becomes "enabled" then the tunnel is activated.
# TUNNEL_STATE enabled
#
# The email addresses below receive diagnostic messages. Separate
# multiple addresses with commas and no white space. The word "none"
# turns off email sending and is the default. It can be overridden by
# using the "-e address" option on the ssh_tunnel command line.
# EMAIL_ADDRESS none
#
# The threshold defined blow is how many failures in a row are required
# before an email is sent. This applies to failed port tests, failed
# ssh connections, and failed scp file transfers.
# EMAIL_THRESHOLD 4 (Number of failures before email is sent)
#
# The loader, tunnel, and pulse programs all have the same sleep value.
# The sleep time can be changed here. It must be in the range of
# 60 to 3600 seconds
# SLEEP_TIME 300
#
# The name below is used by the "pulse" program running on the server to detect
# if the IP address of the client has changed. pulse greps the output of
# netstat looking for this name. It sends an email if it changes. For this
# to work, the /etc/hosts file on the server must have the client's IP address
# associated with this name. Leave the name blank to turn this off.
# NETSTAT_CLIENT_NAME ssh_client_netstat_name
#
########################################################################
# The following section is the ssh config file for the tunnel program.
Host ssh_tunnel
HostName ssh_server_WAN_name
Port = 22
UserKnownHostsFile = /path/to/ssh_clients/file/known_hosts
User = login_username
IdentityFile = /path/to/ssh_clients/file/id_rsa.keyname
Compression = yes
RemoteForward = 55920 machine1_name:5920
RemoteForward = 55925 machine2_name:5925
RemoteForward = 55995 machine3_name:5995
RemoteForward = 50022 ssh_client_name:22
LocalForward = 50022 ssh_server_name:22
########################################################################
Host heartbeat
HostName localhost
Port = 50022
UserKnownHostsFile = /path/to/ssh_clients/file/known_hosts
User = login_username
IdentityFile = /path/to/ssh_clients/file/id_rsa.keyname
EOF
} # make_tunnel_conf
#------------------------------------------------------------------------------
# web_page. Function that creates a web page and copies it somewhere periodically.
# If the file webpage.copy exists then the web page is created and webpage.copy
# is called with the name of the web page in $1. The webpage.copy file is unique
# and must be self contained to copy the file where it needs to be. This can
# be to another drive on the LAN or could be using scp. Use "ssh_tunnel -m" to make
# a template that can be modified for your needs.
function web_page () {
local func_name=web_page
if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> Inside web_page: $WEB_PAGE_UPDATE_CNTR" >> logfile; fi
# Check if webpage.copy exists and exit if it doesn't
if [ ! -e webpage.copy ]; then return; fi
# Getting here means webpage.copy exists
# Only proceed if web page update counter is zero
if [ $WEB_PAGE_UPDATE_CNTR -gt 0 ]; then
let WEB_PAGE_UPDATE_CNTR=WEB_PAGE_UPDATE_CNTR-1
return
else
WEB_PAGE_UPDATE_CNTR=$WEB_PAGE_UPDATE_RATE
fi
# Getting here means it's time to create a new web page
# Build the web page
cat << EOF >| webpage.html
ssh_tunnel statistics: $SSH_CLIENT to $SSH_SERVER
ssh_tunnel statistics: $SSH_CLIENT to $SSH_SERVER
Time: `date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`
Current tunnel state: $TUNNEL_STATE Operating Mode: $OPERATING_MODE
Overall Statistics since $LDR_STATS_CLEARED
EOF
# Output the overall statistics
cat loader.stats >> webpage.html
cat tunnel.stats >> webpage.html
echo "
" >> webpage.html
if [ -e pulse.stats ]; then
echo "Contents of pulse.stats
" >> webpage.html
echo "" >> webpage.html
cat pulse.stats >> webpage.html
echo "
" >> webpage.html
fi
echo "Current users, uptime, and load average
" >> webpage.html
echo "" >> webpage.html
$W >> webpage.html
echo "
" >> webpage.html
echo "Current output of ps-ef
" >> webpage.html
echo "" >> webpage.html
case $OS_TYPE in
5.8*) # Solaris 8
$PS -ef >> webpage.html
;;
2.4*) # Mandrake Linux 8.x and 9.x, Redhat Linux Work Station Enterprise 3
$PS -ef >> webpage.html
;;
2.6*) # Redhat Linux Work Station Enterprise 4
$PS -ef >> webpage.html
;;
1.5*) # Cygwin: Requires procps package installed
$PS ax >> webpage.html
;;
*) # Unknown OS
echo "ERROR [$LINENO]$func_name> Unknown OS"
exit
;;
esac
echo "
" >> webpage.html
echo "Current output of netstat showing tcp connections
" >> webpage.html
echo "" >> webpage.html
$NETSTAT >> webpage.html
echo "
" >> webpage.html
echo "Contents of /etc/hosts file
" >> webpage.html
echo "" >> webpage.html
cat /etc/hosts >> webpage.html
echo "
" >> webpage.html
if [ -e loader.conf ]; then
echo "Contents of loader.conf
" >> webpage.html
echo "" >> webpage.html
cat loader.conf >> webpage.html
echo "
" >> webpage.html
fi
if [ -e tunnel.conf ]; then
echo "Contents of tunnel.conf
" >> webpage.html
echo "" >> webpage.html
cat tunnel.conf >> webpage.html
echo "
" >> webpage.html
fi
echo " " >> webpage.html
# Use the webpage.copy script in the back ground to transfer the web page where it needs to go.
./webpage.copy webpage.html &
} # web_page
#------------------------------------------------------------------------------
# set_working_dir: Function that sets the WORKING_DIR based on $0
function set_working_dir () {
# Configure working directory
WORKING_DIR=`dirname $0`
if [ "$WORKING_DIR" = "." ]; then
WORKING_DIR=`pwd`
fi
cd $WORKING_DIR
} # set_working_dir
#------------------------------------------------------------------------------
# cron_and_daemon: Function used by cron and daemon processing
# $1: cron or loader
function cron_and_daemon () {
if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside cron_and_daemon" >> logfile; fi
# Check the tunnel state
read_loader_conf
read_ldr_stats
read_tnl_stats
read_tunnel_conf
if [ "$TUNNEL_STATE" = "disabled" ]; then
# tunnel state is disabled so check server to see if tunnel.conf has been updated
# This is a good place to remove left over *.ret_val files since nothing
# else is running.
rm -f *.ret_val
# Check ssh server port and update statistics
test_port $SERVER_DOMAIN_NAME $SERVER_DOMAIN_PORT &>/dev/null
test_port_stats "$RC_TEST_PORT"
if [ "$RC_TEST_PORT" -eq "$GOOD_PORT" ]; then
# Port is good so check for updated tunnel.conf file
update_file $SSH_SERVER_DIR/tunnel.conf 2>/dev/null
update_file_stats "$RC_UPDATE_FILE"
if [ "$RC_UPDATE_FILE" -eq $UPDATED ]; then
# Read tunnel.conf since an updated version was just received. Need to set the ARG_?
# flags FALSE in case they are leftover true from startup so that tunnel.conf values
# will be read and used.
ARG_E=FALSE
ARG_ST=FALSE
read_tunnel_conf
write_ldr_stats
if [ "$TUNNEL_STATE" = "enabled" ]; then
log_event "[$LINENO] $1: Tunnel state changed to enabled. Starting loader."
send_ldr_email "[$LINENO] $1: Tunnel state changed to enabled. Starting loader."
# Set loader heartbeat to RUNNING since we're starting up a new loader
echo "RUNNING" >| loader.heartbeat
# Start the loader
$WORKING_DIR/ssh_tunnel -d $D -p loader -st $SLEEP_TIME -wr $WEB_PAGE_UPDATE_RATE -e $EMAIL_ADDRESS &
return
else
# tunnel state still isn't enabled so simply return after updating webpage.
# Update the webpage every time through cron or daemon when the state is disabled.
# This is done in the loader when the tunnel is enabled.
WEB_PAGE_UPDATE_CNTR=0
web_page
# Check if the logfiles need to be rotated
logfile
return
fi
else
# File not updated so just return after updating the webpage.
# Update the webpage every time through cron or daemon when the state is disabled.
# This is done in the loader when the tunnel is enabled.
WEB_PAGE_UPDATE_CNTR=0
web_page
# Check if the logfiles need to be rotated
logfile
return
fi
else
# Port is bad so don't even try to continue
# Check if the logfiles need to be rotated
logfile
return
fi
else
# Tunnel state is enabled. Check if the loader pid is still active.
get_loader_pid
test_pid "$RC_GET_LOADER_PID"
if [ "$?" -eq $ACTIVE ]; then
# Getting here means the tunnel is enabled and the loader pid is still active.
# Get contents of loader.heartbeat or initialize the file if it doesn't exist.
if [ -e loader.heartbeat ]; then
HEARTBEAT=`cat loader.heartbeat`
else
echo "RUNNING" >| loader.heartbeat
HEARTBEAT="RUNNING"
fi
# Kill running process if loader.heartbeat indicates the thread is stalled.
# The cron or daemon task writes NOT_RUNNING to loader.heartbeat. The loader
# while loop overwrites this with RUNNING. The time between cron or daemon runs
# must be longer than the loader sleep rate for this to work.
if [ "$HEARTBEAT" == "RUNNING" ]; then
# Process still running. Overwrite heartbeat and return.
echo "NOT_RUNNING" >| loader.heartbeat
return
else
# loader has stalled. Log and event, send an email, kill the stalled process, restart the loader, and return.
log_event "[$LINENO] $1 processing: loader.heartbeat is NOT_RUNNING. Loader stalled. Killing loader (pid $RC_GET_LOADER_PID) and restarting loader."
send_ldr_email "[$LINENO] $1 processing: loader.heartbeat is NOT_RUNNING. Loader stalled. Killing loader (pid $RC_GET_LOADER_PID) and restarting loader."
kill -9 $RC_GET_LOADER_PID
# Set loader heartbeat to RUNNING since we're starting up a new loader
echo "RUNNING" >| loader.heartbeat
# Restart the loader
$WORKING_DIR/ssh_tunnel -d $D -p loader -st $SLEEP_TIME -wr $WEB_PAGE_UPDATE_RATE -e $EMAIL_ADDRESS &
return
fi
else
# Tunnel state is enabled but the loader pid is inactive. Need to restart the loader.
log_event "[$LINENO] $1 processing: loader pid $RC_GET_LOADER_PID inactive. Restarting loader."
send_ldr_email "[$LINENO] $1: no active loader. Restarting."
# Set loader heartbeat to RUNNING since we're starting up a new loader
echo "RUNNING" >| loader.heartbeat
# Restart the loader
$WORKING_DIR/ssh_tunnel -d $D -p loader -st $SLEEP_TIME -wr $WEB_PAGE_UPDATE_RATE -e $EMAIL_ADDRESS &
return
fi
fi
} # cron_and_daemon
#------------------------------------------------------------------------------
# main: Function that is run first. Pass command line to it with $*
function main () {
if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$PROGRAM_NAME:$$> Inside main" >> logfile; fi
# If no arguments are passed then output the help screen and exit
if [ $# -eq 0 ]; then
display_help
exit
fi
# Set the debug level
set_debug_level
# Setup the environment
env_setup
# Process the command line
process_command_line $*
# Check if this is the pulse program
if [ ${PROGRAM_NAME:-""} = pulse ]; then
pulse
exit
fi
# Check for existence of required loader.conf file
if [ ! -e loader.conf ]; then
display_help
echo "ERROR [$LINENO]: No loader.conf file in $WORKING_DIR"
log_event "ERROR [$LINENO]: No loader.conf file in $WORKING_DIR"
send_ldr_email "[$LINENO] No loader.conf file in $WORKING_DIR"
exit
fi
# Check if operating in daemon mode
if [ "$DAEMON_MODE" = "TRUE" ]; then
# Set the operating mode for web_page output
export OPERATING_MODE="daemon"
# Set PROGRAM_NAME to daemon so that debug lines have a name
PROGRAM_NAME="daemon"
# Save the daemon mode pid for later testing by /etc/init.d/ssh_tunnel
echo $$ >| ssh_tunnel.pid
# Initialize all the environment variables. Order is important.
read_loader_conf
read_ldr_stats
read_tnl_stats
read_tunnel_conf
# Kill "tunnel" and ssh link if active since a new loader is going to start up
get_tunnel_pid
test_pid $RC_GET_TUNNEL_PID
if [ "$RC_TEST_PID" -eq $ACTIVE ] && [ "$TEST_PID_PGM" = "ssh_tunn" ]; then
log_event "[$LINENO] daemon: Active tunnel $RC_GET_TUNNEL_PID detected. Killing it."
let TUNNEL_KILLED=TUNNEL_KILLED+1
write_ldr_stats
send_ldr_email "[$LINENO] daemon: Killed active tunnel: $RC_GET_TUNNEL_PID"
TUNNEL_LAST_KILLED=`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`
write_ldr_stats
kill -9 $RC_GET_TUNNEL_PID 2>/dev/null
kill_ssh_link
fi
# Stay in this loop forever
while [ true ]; do
# Check if the debug level has changed while sleeping
set_debug_level
log_event "[$LINENO] daemon processing using -D $DAEMON_SLEEP"
cron_and_daemon "daemon"
sleep $DAEMON_SLEEP
done
fi
# Perform cron processing if -c was on command line
if [ "$CRON_JOB" = "TRUE" ]; then
# Set the operating mode for web_page output
export OPERATING_MODE="cron"
log_event "[$LINENO] Cron processing using -c"
cron_and_daemon "cron"
exit
fi # End of cron processing
# Check for program name to execute
if [ "$PROGRAM_NAME" = "" ]; then
display_help
echo "ERROR [$LINENO]: You must provide program name using -p"
exit
else
case $PROGRAM_NAME in
loader) #loader_script processing
loader;;
tunnel) #tunnel processing
tunnel;;
*) #error case
display_help
echo "ERROR [$LINENO]: Unrecognized program name passed with -p"
exit
esac
fi
} # main
# Set working directory, get version, set debug level, and then invoke the main function
set_working_dir
get_version
set_debug_level
main $*