#!/usr/bin/perl
# YABS - Yet Another Backup Script
# Edit path on the first line to where perl is on your machine.  
# The line shown is a typical location on linux and cygwin systems.
# On Solaris perl might be in /usr/local/bin/perl
#
# Execute "yabs.pl -h" for help
#
# Copyright: 2003-2010 by John R Larsen  -  john@larsen-family.us
# Released under the same Artistic license as perl
# Note: Earlier versions of this script were called "backup" and "backupl.pl"
#
# 180911 jrl: Had to tweak the following line adding the "./" to the $file. The latest
#             cygwin didn't handle it properly without the "./" in front of the name.
#        unless ($return = do "./$file") {

# $Id: yabs.pl,v 3.18 2018-09-11 11:45:00 jlarsen Exp $
use strict;
use warnings;
use Errno qw(EAGAIN);
use File::Basename;
use File::Copy;
use Getopt::Long qw(:config permute pass_through);
use POSIX ":sys_wait_h";

#------------------------------------------------------------------------------
# Define program wide variables here

# Constants defined here
use constant GOOD_PORT => 1;
use constant BAD_PORT => 0;
use constant TRUE => 1;
use constant FALSE => 0;


# Internal program wide variables declared here
chomp (my $backup_yymmddhhmm = `date +%y%m%d-%H%M`);
chomp (my $backup_date = `date +%y%m%d`);
chomp (my $year = `date +%Y`);
chomp (my $month = `date +%m`);
chomp (my $day = `date +%d`);
my $bad_port_cntr = 0;
my $compress_rsync_backup = "no";
my $debug_flags = 0;
my $dest_dir;
my $fail_safe = 50;
my $func_name = "none";
my $host;
my $old_debug_flags;
my $rc;
my $recurse_level;
my $rsync_opts = "";
my $script_ver;         # Initialized by get_version
my $script_date;        # Initialized by get_version
my @search_err;
my $search_err_index = 0;
my $short_backup_dir;
my $webpage_local_dir = "none";
my $webpage_remote_dir = "none";
my $working_dir;


# Command line argument variables
my $yabs_dirlist = 'yabs.dirlist';
my $yabs_lock = 'yabs.lock';
my $backup_type = 'none';
my $baseline_day = "01";
my $D = 0;              # Debug flags. Default is 0.
my $delta_days = 1;
my $email_address = 'none';
my $keep_days = 0;
my $fix_files = FALSE;
my $test_mode = FALSE;


# Global variables that are set by values in yabs.conf
# Global variables names are in uppercase
our $DF_PGM = "not set";
our $HOST_PGM = "not set";
our $LS_PGM = "not set";
our $MAIL_PGM = "not set";
our $PS_PGM = "not set";
our $RSYNC_PGM = "not set";
our $SCP_PGM = "not set";
our $TELNET_PGM = "not set";
our $SSH_PGM = "not set";
our $W_PGM = "not set";
our $ZIP_PGM = "not set";


#--- FUNCTIONS START HERE ARRANGED ALPHABETICALLY (except "main" is at end) ---

#------------------------------------------------------------------------------
# display_debug_help:  Function to display debug help.
sub display_debug_help 
{
   my $func_name = "display_debug_help";
   if ($D & 0x1) {printf ("[%d]$func_name:$$> Inside display_debug_help\n", __LINE__);}

   # Pipe this help screen through more so it doesn't fly off the screen
   open MORE, "|more" or die "cannot pipe through more: $!";
   print MORE << "EOF";
----- YABS Debug Help ($script_ver $script_date) ------------------------------
Debug is enabled in one of three ways in descending order of precedence: 

   "-debug 0xNNNN" on command line
   DEBUG_FLAGS environment variable
   "debug_flags" file in same directory as yabs.pl

Debug output can be changed "on the fly" by changing the contents of
"debug_flags", which is read each time yabs goes through its while loop.

Debug output is added only to the logfile.  If you want to see in real time
what is going on then tail the logfile with this command: "tail -f logfile".
If output is redirected to STDOUT then debug also goes to STDOUT.

0000_0000 - No debug

0000_0001 - Enable all the "Inside function name" debug lines
0000_0002 - read_yabs_dirlist: Display contents of yabs.dirlist file
0000_0004 - Display environment after running setup_env
0000_0008 - webpage: verbose output of scp webpage transfer

0000_0010 - test_port: Display data from telnet connection
0000_0020 - recurse_tree_rsync: Display recurse level
0000_0040 - recurse_tree_rsync: Display contents of current directory
0000_0080 - recurse_tree_rsync: Display rsync command line used

0000_0100 - read_yabs_dirlist: Display search_err terms found
0000_0200 - main: Display error conditions found in logfile
0000_0400 - purge_archives: Display name of deleted archives

8000_0000 - Redirect all output to STDOUT instead of "logfile"


Copyright: 2003-2010 by John R Larsen  -  john\@larsen-family.us
Released under the same Artistic license as perl

EOF
   close MORE;
   exit 0;
} # display_debug_help


#------------------------------------------------------------------------------
# display_env:  Function that displays current environment values
sub display_env 
{
   my $func_name = "display_env";
   if ( $D & 0x1 ) { printf ("[%d]$func_name:$$> Inside display_env\n", __LINE__) }
   printf ("----- CURRENT ENVIRONMENT VARIABLE VALUES -----------------\n");
   printf ("yabs_dirlist            %s\n", $yabs_dirlist);
   printf ("yabs_lock               %s\n", $working_dir/$yabs_lock);
   printf ("backup_type             %s\n", $backup_type);
   printf ("backup_yymmddhhmm       %s\n", $backup_yymmddhhmm);
   printf ("baseline_day:           %s\n", $baseline_day);
   printf ("debug flags:            0x%x\n", $D);
   printf ("delta_days:             %d\n", $delta_days);
   printf ("email_address:          %s\n", $email_address);
   printf ("keep_days:              %s\n", $keep_days);
   printf ("compress_rsync_backup:  %s\n", $compress_rsync_backup);
   printf ("fail_safe:              %s\n", $fail_safe);
   printf ("fix_files:              %s\n", $fix_files);
   printf ("test_mode:              %s\n", $test_mode);
   printf ("webpage_local_dir:      %s\n", $webpage_local_dir);
   printf ("webpage_remote_dir:     %s\n", $webpage_remote_dir);
   printf ("working_dir:            %s\n", $working_dir);

} # display_env


#------------------------------------------------------------------------------
sub display_help 
{
my $func_name = "display_help";
if ($D & 0x1) {printf ("[%d]$func_name:$$> Inside display_help\n", __LINE__);}

# Pipe this help screen through more so it doesn't fly off the screen
open MORE, "|more" or die "cannot pipe through more: $!";
print MORE << "EOF";
----- YABS - Yet Another Backup Script ($script_ver $script_date) -------------------
usage:

     yabs.pl [ options ]

Options:
-bd n    Baseline day for delta backups. (default: 1, meaning 1st of month.)
         (0 forces a full backup regardless of current day)
-d 0xN   Debug flags value.  Use the "-hd" option to see how to use them.
-dd n    Delta Days.  Number of days included in a delta archive (Default: 1)
-e adr   Override email address setting in yabs.dirlist (Default is none)
         (Separate multiple email addresses with commas and no spaces)
-f file  Name of yabs.dirlist file (default: yabs.dirlist)
-fs n    FailSafe recursion level (default: $fail_safe)
-h       Help screen
-hd      Help Debug. This describes how the builtin debugging works.
-hs      Help setup. This gives detailed operating and setup instructions.
-kd n    Keep Days. Delete archives older than n days. (default 0: keep all)
-mb      Make a "yabs.dirlist" file (won't overwrite existing file)
-mdf     Make a yabs.df file
-r       Rename (copy) "backup.*" files to "yabs.*"
-t type  Override the yabs type defined in the yabs.dirlist file
-test    Test mode. This shows what would have been done.
         (All output is sent to STDOUT when running in test mode.)

Copyright: 2003-2010 by John R Larsen  -  john\@larsen-family.us
Released under the same Artistic license as perl
EOF
   close MORE;
   exit 0;
} # display_help


#------------------------------------------------------------------------------
# display_setup_help:  Function to display help on ssh_tunnel configuration.
sub display_setup_help 
{
   my $func_name = "display_setup_help";
   if ($D & 0x1) {printf ("[%d]$func_name:$$> Inside display_setup_help\n", __LINE__);}

   # Pipe this help screen through more so it doesn't fly off the screen
   open MORE, "|more" or warn "cannot pipe through more: $!";
   print MORE << "EOF";
DESCRIPTION:
YABS stands for "Yet Another Backup Script."  yabs performs backups using the
rsync or zip utilities.  The file yabs.dirlist controls operation of yabs.pl.
Default values for options are set there.  yabs.dirlist also contains a list of
starting directories.  Read the yabs.dirlist file for a description of all
these options.

yabs.pl supports three different types of backups: daily, delta, and rsync,
which are described below.

yabs.pl starts in the directory specified in the yabs.dirlist file.  It
recurses the directory structure from that point.  In "rsync" mode it looks for
a file named yabs.rsync.  In all other modes it looks for a file named
yabs.list.  In rsync mode when yabs.pl finds an instance of yabs.rsync in a
directory, it starts a recursive rsync operation in that directory.  In other
modes when yabs.pl finds a yabs.list file it uses the contents of that file as
an input file list to the zip utility to create a zip archive.  Once the zip or
rsync operation is done, yabs.pl exits that directory and continues recursing
through the rest of the directory structure.  Once the entire directory
structure has been searched yabs.pl moves on to the next starting directory
defined in yabs.dirlist.

daily: 
This uses the zip utility. It refreshes the destination zipfile every time it
runs.  The name of the zip file reflects the starting point of the backup.
For example, if the yabs.list file is found in "/home/jlarsen/my music" the
zipfile has the name "home.jlarsen.my^music.zip".  A dot (.) is placed between
directory names and spaces are replaced with a circumflex (^).

delta: 
This uses the zip utility.  It creates a full zipfile on the baseline day
(default 1st of the month). On other days it creates a zipfile of only the
files that have been modified during the previous number of days specified by
the option -dd.  The default is 1 day.  Filenames for delta zipfiles are
constructed the same way "daily" filenames are made, but a date stamp is added
at the end.  Date stamps look like "__YYMMDD-fb" or "__YYMMDD-pbN".  The "-fb"
indicates a "full backup" was done.  The "-pbN" indicates a partial backup was
done and "N" is the number of days included in the partial backup.  For
example, if the yabs.list file is found in "/home/jlarsen/my music" and a full
backup is done on May 4, 2009, the zipfile has the name:
"home.jlarsen.my^music__090504-fb.zip" If "DELTA_DAYS" is set to 7 in the
dirlist file or if "-dd 7" is put on the command line then the file has the
name: "home.jlarsen.my^music__090504-pb7.zip"

rsync: 
This uses the rsync utility.  The destination of the backup can be on a local
drive or on a remote machine.  SSH is used to make a secure connection to a
remote machine.  In rsync mode yabs.pl creates a copy of the directory
structure.  In rsync mode the START_DIR directory in the yabs.dirlist file is
considered the root.  When yabs.pl finds the yabs.rsync file in a directory it
uses that as the starting point.  The full path is maintained in the backup.
For example, if you are running yabs.pl on a computer named "screamer" and you
are backing up to a remote computer named "zippy", you could put a DEST_DIR
directory line in yabs.dirlist of "zippy:/backups/screamer".  When yabs.pl
reads this line it would then look for a host entry of "zippy" in the host.conf
file.  yabs.pl uses the settings for "zippy" to do the remote rsync backup.
Now, if on screamer yabs.pl found a yabs.rsync file in the /var/mail directory
yabs.pl would use rsync to copy all the files in /var/mail over to zippy into
the /backups/screamer/var/mail directory.


EMAIL:
yabs.pl sends email to email addresses defined in yabs.dirlist.  The -e option
on the command line can also be used and overrides the value in yabs.dirlist.
Multiple email addresses can be used by separating them with commas and no
white space.  The email reports all aspects of the backup including date,
times, changed files, and beginning and ending disk usage.  A single email is
sent at the end of the entire yabs operation.


WEBPAGE UPDATE:
yabs.pl can update a webpage.  The webpage can be copied to a directory on the
local machine or it can be copied to a remote webserver.  Directory paths in
yabs.dirlist specify where the webpages go.  If enabled, the webpage is updated
just before each zip or rsync operation begins.  That makes it possible to
monitor progress on long yabs operations by looking at the webpage.  The
webpage is essentially an incrementally updated duplicate of the contents of
the final email.


CRON OPERATION:
It is intended that yabs.pl be run as a cron task.  A typical crontab line
using default values is shown below that runs yabs starting at 3:01 AM.  
   01 03 * * * /root/yabs/yabs.pl
The example below uses a yabs.dirlist with a different name
   01 03 * * * /root/yabs/yabs.pl -f rsync.dirlist


ERROR CHECKING:
yabs.pl can be run in "test mode" by putting "-test" on the command line.  In
this mode it does everything except the actual time consuming archive
operations.  It also outputs a diagnostic message when it finds each control
file.  This makes it possible to verify that the yabs configuration operates
as expected.


CONTROL FILES:
Operation of yabs.pl is controlled by several files.  These files are
described below:

yabs.dirlist:
The format of this file is described in the file itself.  This file defines the
starting directories, destination storage directories, and commands to execute
before starting the yabs operation.  It also has the values used by many of
the program options.  yabs.pl automatically makes a yabs.dirlist file
when it is run for the first time.  Use "yabs.pl -mb" to create additional
files file that you can edit for your situation.  Examples of "commands to
execute" before a yabs operation starts might be like starting a ClearCase
view to make the starting directory visible, or mounting a drive.  Spaces in
directory names are permissible.  There is no need to put double quotes around
the names.  yabs.pl takes care of that.  It is best to create a separate
yabs.dirlist file and rename it for each type of backup that you want to
run.  Use the "-f filename.dirlist" command to direct yabs.pl to use a
specific file.  For example, a file named "rsync.dirlist" would be a good name
for an rsync type of backup.  "delta.dirlist" would be a good name for delta
backups.

yabs.list:
When yabs.pl finds a yabs.list file in a directory, it stops recursing past
that point, and uses its contents to define which files zip will archive.  Each
line of this file defines a directory or file that should be included in the
archive.  A single "." in the file directs zip to recursively archive
everything in the current directory.  This is the easiest way to archive a
whole directory structure.  If you only want to archive some of the files or
directories in the current directory, then put each name on a separate line in
the file.  This is useful if you want to include files in a directory that
isn't a subdirectory of the current directory.  yabs.list is not used in
"rsync" mode.

yabs.enter_cmds:
When yabs.pl finds this file it executes the commands in the file.  The pwd
used is the directory where the file is located.  An example of using this file
is to remove temporary files in a directory before archiving.  For example, the
Quicken program makes temporary files that start with tilda "~".  These files
mess up the zip program.  Putting a command like 
   "rm -f /home/jlarsen/quicken/\~*" 
in yabs.enter_cmds would remove all the temp files starting with tilda.

yabs.exit_cmds:
When yabs.pl finds this file it executes the commands in the file just
before leaving the directory.  When this script is called, \$1 holds the path
to the archive directory and \$2 holds the name of the archive just made.  If
these values are null then no archive was made in the directory where
yabs.exit_cmds is located.

yabs.no_recurse:
When yabs.pl finds this file it exits the directory immediately doing nothing
else.  This is useful if there is a directory that you don't want yabs.pl to
recurse into.  For example, there are many hidden directories in a user's home
directory (directories that start with a single dot ".") that may have
extensive subdirectories.  Putting yabs.no_recurse keeps yabs from wasting time
and resources.

yabs.rsync:
The yabs.rsync file is looked for when yabs is run in "rsync" mode.  When
found, yabs stops recursing past that point.  It uses the parameters in the
host.conf file to perform an rsync copy.  The destination directory on the
remote host comes from the DEST_DIR value in the yabs.dirlist file.  The local
directory is replicated in the DEST_DIR directory.  rsync options are defined
in yabs.dirlist.

host.conf:
This file is used in rsync mode and contains ssh "host" definitions for remote
sites where backups are stored.  The format of this file is described in the
file itself.  It is created automatically when yabs.pl is run for the first
time.

logfile:
This contains all the output of the last run of yabs.pl.  It is over written
each time yabs.pl runs.  The contents of logfile are used in the email sent
and in the webpages.


CONFIGURING SSH CONNECTIONS:
Read the instructions in host.conf.


SUPPORTED PLATFORMS:
This script has been used successfully on the following systems:
Solaris 8 using bash v2.03.0(1)
Solaris 9 using bash v2.05.0(1)
Mandrake Linux 9.1 using bash v2.05b.0(1)
cygwin 1.5.x and later using bash v2.05b.0(1) and later
Ubuntu 8.10 Linux 2.6.27-11-generic using bash v3.29(1)

Copyright: 2003 through 2010 by John R Larsen
Free for personal use.  Contact the author for commercial use.
http://larsen-family.us            john\@larsen-family.us
EOF
   close MORE;
   exit 0;
} # display_setup_help


#-------------------------------------------------------------------------------
# get_version.  Function that parses script version and date info from the RCS Id line.
sub get_version 
{
   my $Id;
   # The cvs version string is embedded in $version_info below
   my $version_info = '$Id: yabs.pl,v 3.19 2018-09-11 11:45:00 jlarsen Exp $';
   # Extract the version and date from the string
   $version_info =~ m{v ([\d\.]+) (\d+.\d+.\d+)};
   #                  ^^                         Match the letter v followed by a space "v "
   #                     ^^^^^^^                 Match any number of digits and periods
   #                             ^               Match a single space
   #                               ^^^^^^^^^^^   Match any number of digits followed by any character
   #                                             followed by any number of digits and any character
   #                                             followed by any number of digits.  YYYY/MM/DD
   $script_ver = "v".$1;  # Stick the letter "v" to the front of the the version number
   $script_date = $2;
} # get_version


#-------------------------------------------------------------------------------
# get_time.  Function that returns date and time
sub get_time 
{
   chomp (my $time = `date '+%Y\-%b\-%d\ %H\:%M\:%S\ %a'`);
   return $time;
} # get_time


#------------------------------------------------------------------------------
# init_debug:  Initialize debug.  Debug flags can be set on the command line,
# through and environment variable, or through the file debug_flags in that
# order of precedence.  The default destination for debug is into the file
# "logfile".  If bit 0x8000_0000 is set then all output is sent to STDOUT
# instead.
sub init_debug 
{
   my $func_name="init_debug";

   # Read value from file debug_flags and store it away
   if (-s "debug_flags") {
      open (DEBUG_FLAGS, "<", "debug_flags") 
         or die "Can't open debug_flags file: $!";

      # Read the value from the file, convert to a hex number and save for later use
      $old_debug_flags = <DEBUG_FLAGS>;
      close DEBUG_FLAGS;
      chomp $old_debug_flags;
      $old_debug_flags = oct $old_debug_flags;
      $debug_flags = $old_debug_flags;
      $D = $debug_flags;
   } else {
      $old_debug_flags = "0";
      $debug_flags = "0";
      $D = 0;
   }

   # Check if DEBUG_FLAGS env variable exists and use it if it does
   if (exists $ENV{DEBUG_FLAGS}) {
      $D = oct $ENV{DEBUG_FLAGS};
   }

   # Setup where log output goes
   &set_log_dest;

} # init_debug


#------------------------------------------------------------------------------
# make_yabs_dirlist: Function to create yabs.dirlist file.  
sub make_yabs_dirlist 
{
   my $func_name = "make_yabs_dirlist";
   if ($D & 0x1) { printf ("[%d]$func_name:$$> Inside make_yabs_dirlist\n", __LINE__) }

   # Don't overwrite an existing file.
   if (-e "yabs.dirlist") {
      unlink "$working_dir/$yabs_lock";
      die "ERROR: yabs.dirlist already exists. Rename it and try again.\n";
   }

   open YABS_DIRLIST, ">", "yabs.dirlist" or die "Cannot open yabs.dirlist for writing: $!";
   print YABS_DIRLIST << "EOF";
# NOT CONFIGURED - DELETE THIS LINE ONCE THIS FILE HAS BEEN CONFIGURED
# Copyright 2003 through 2010 John R Larsen - john\@larsen-family.us
#----- YABS - Yet Another Backup Script ($script_ver $script_date) -------------------

#----------------------------------------------------------------------------
# The type of backup performed is specified here.  Allowable values
# are: rsync, daily, or delta
#
BACKUP_TYPE     none


#----------------------------------------------------------------------------
# The email addresses below receive yabs script output.  Separate
# multiple addresses with commas and no white space.  The word "none"
# turns off email sending and is the default.
#
EMAIL_ADDRESS     none


#----------------------------------------------------------------------------
# The number of days to keep backups.  Zero (0) keeps all backups.
#
KEEP_DAYS     0


#----------------------------------------------------------------------------
# The "baseline" day for delta backups.  On this day of the month a full
# backup will be performed.  A value of zero (0) here forces a full backup
# regardless of the day
#
BASELINE_DAY     1


#----------------------------------------------------------------------------
# The number of days to include in a delta zipfile.  Files modified within
# this number of days are included in the date stamped zipfile.
#
DELTA_DAYS     1


#----------------------------------------------------------------------------
# WEBPAGE UPDATES
# Webpages can be created and copied to a local directory on the machine
# where yabs.pl is running and/or to a directory on a remote webserver.
# A value of "none" turns off the webpage copy.  If both are "none" then
# webpage creation is disabled.  Put a valid directory in place of "none" 
# to enable this feature.  Note that for remote webpage updates to work
# a host entry for "webpage" needs to be in the host.conf file.
#
WEBPAGE_REMOTE_DIR  none
WEBPAGE_LOCAL_DIR   none


#----------------------------------------------------------------------------
# If the rsync option "--backup" is included then deleted or modified files
# are automatically moved into the directory "DEST_DIR.YYMMDD-HHMM". Setting
# the following to "yes" will zip that directory into a file instead of a
# directory structure so that it takes up less space on the drive. Setting
# it to "no" will leave the backup files as a directory structure.
COMPRESS_RSYNC_BACKUP   no


#----------------------------------------------------------------------------
# RSYNC OPTIONS (Do not delete or modify this line)
# This section allows you to customize the options passed to rsync.
# Non comment lines between the "# RSYNC OPTIONS" and the "# END RSYNC OPTIONS"
# line are concatenated to make up the options passed to rsync.
# rsync has a very large number of options.  Use "man rsync" to understand 
# those used here.
# -r, --recursive             recurse into directories
# -l, --links                 copy symlinks as symlinks
# -t, --times                 preserve modification times
# -D                          same as --devices --specials
# -u, --update                skip files that are newer on the receiver
# -v, --verbose               increase verbosity
# -z, --compress              compress file data during the transfer
# -R, --relative              use relative path names
# -s, --protect-args          no space-splitting; wildcard chars only
#     --rsync-path            put in path to rsync on remote host
#     --filter                used to include or exclude files and directories
#     --backup                preserve modifed or deleleted files
-rltDuvzRs 
--rsync-path="/usr/bin/rsync" 
# Add filter lines as needed to avoid backing up junk that you don't need
# to preserve. Some examples are shown below.
--filter='exclude Temporary Internet Files' 
--filter='exclude Application Data' 
--filter='exclude Cookies'
--no-g 
--chmod=ugo=rwX 
--delete 
--timeout=60
# Remove or comment out the --backup option below to prevent saving deleted 
# and modified files.  If --backup is included here then yabs.pl 
# automatically adds this option: --backup-dir=DEST_DIR.YYMMDD-HHMM
# All modified or deleted files from the previous time yabs.pl was run
# are moved into the --backup-dir directory.
--backup 
# Note: If "-test" is included on the yabs.pl command line then the rsync
# option --dry-run is automatically added.
# Note: If the DEST_DIR has a "host:" indicating a remote host, then the 
# following option is automatically added:
#      "-e \$SSH_PGM -F \$working_dir/host.conf"
# The above option directs rsync to use SSH as its transport.
# END RSYNC OPTIONS (Do not delete or modify this line)

#----------------------------------------------------------------------------
# ERROR REPORTING (Do not delete or modify this line)
# This section allows you to specify patterns to look for at the end of the
# backup to determine if an error has occurred during the backup.  If these
# patterns are detected then the term "ERROR!" will appear in the subject
# line of the email and at the beginning of the webpage.  Put only one search
# term per line.  Comment lines starting with the pound sign (#) are ignored.
# Standard perl pattern matching metacharacters and regex are supported.
# Two default values are included.
^ERROR.*
^rsync error:.*
# END ERROR REPORTING (Do not delete or modify this line)

#----------------------------------------------------------------------------
# This section specifies the parameters for each yabs operation. The order
# of lines is important.  Each entry consists of 4 lines that are described
# below.  Each line starts with a name.  Only modify the text after the 
# colon.
# 
# Line 1: PRE_CMDS:
# The first line contains commands to be executed before cd'ing to
# the start directory.  Put all commands on one line separating them with
# semicolons.  The starting directory might not be available without these
# commands.  For example, if the directory is in a ClearCase view, the view
# might need to be started before the directory is available.  If no commands
# are needed, then put nothing after the colon.
# 
# Line 2: START_DIR:
# The second line is the starting directory.  yabs.pl starts here and
# recurses the entire directory structure looking for the file yabs.list
# or yabs.rsync.  If found, the contents of yabs.list are used to tell 
# yabs.pl what to archive for daily and delta backups.
# (Use "yabs -h" to read more about files yabs.list,
# yabs.enter_cmds, yabs.exit_cmds, and yabs.no_recurse)
# Spaces in directory names are permissible.  There is no need to put 
# double quotes around the names.  yabs.pl takes care of that.
# 
# Line 3: DEST_DIR:
# The third line is the directory where archives are stored.
# Spaces in directory names are permissible.  There is no need to put 
# double quotes around the names.  yabs.pl takes care of that.  A remote
# host name can be used for rsync operation.  Leave off the "host:" if
# the backup is to a local drive.
# 
# Line 4: POST_CMDS:
# The fourth line contains commands that need to be executed after 
# yabs.pl completes.  Put all commands on one line separating them with
# semicolons.  If no commands are needed, then delete everything after 
# the colon.
#
# Use dashed lines to group the four lines. Comment lines are permissible
# between the four lines. Explaining commands will make debugging easier
# later on.
# 
# IMPORTANT: Make sure there are NO trailing spaces or slashes, else the
# backup will fail.  
# 
# !!! TEST YOUR FILE !!!
# Test your yabs.dirlist file using "yabs.pl -test" and carefully
# look at the generated output for correct path names.  yabs.pl will
# warn you of any trailing slashes or spaces.
# 
# Below is a commented example of an entry. Change entries as needed.
#----------------------------------------------------------------------------
PRE_CMDS:  separate commands; with semicolons
START_DIR: /path/to/directory/to/start/in
DEST_DIR:  host:/path/to/directory/where/archives/are/stored
POST_CMDS: separate commands; with semicolons
#----------------------------------------------------------------------------

EOF
# The following line must be used if --backup is found in this file.
   close YABS_DIRLIST;
   print STDERR "Created yabs.dirlist file.  You must edit this file with your settings.\n"
} # make_yabs_dirlist


#------------------------------------------------------------------------------
# make_yabs_df: Function to create a template yabs.df script
sub make_yabs_df
{
   my $func_name = "make_yabs_df";
   if ($D & 0x1) { printf ("[%d]$func_name:$$> Inside make_yabs_df\n", __LINE__) }

   # Don't overwrite an already existing file
   if (-e "yabs.df") {
      return 0;
   }

   # Getting here means yabs.df doesn't exist and needs to be created
   open YABS_DF, ">", "yabs.df" or die "Cannot open yabs.df for writing: $!";
   print YABS_DF << "EOF";
#!/bin/bash
# Copyright 2003 through 2010 John R Larsen - john\@larsen-family.us
#----- YABS - Yet Another Backup Script ($script_ver $script_date) -------------------
# This is the yabs.df script called by "yabs.pl" to get a listing of disk
# usage.  You don't need this script if the normal output of df is acceptable.
# If this script doesn't exist then "yabs" will use df and capture its
# output.  If you want to limit the output of df to a few specific directories
# then modify this script.

$DF_PGM
EOF

   # Set execute permissions
   chmod 0755, "yabs.df";

   close YABS_DF;
   print STDERR "Created yabs.df file.  You must edit this file with your settings.\n"
} # make_yabs_df


#------------------------------------------------------------------------------
# make_host_conf: Function to create host.conf file.  
sub make_host_conf 
{
   my $func_name = "make_host_conf";
   if ($D & 0x1) { printf ("[%d]$func_name:$$> Inside make_host_conf\n", __LINE__) }

   # Don't overwrite an already existing file
   if (-e "host.conf") {
      return 0;
   }

   # Getting here means host.conf doesn't exist and needs to be created
   open HOST_CONF, ">", "host.conf" or die "Cannot open host.conf for writing: $!";
   print HOST_CONF << "EOF";
#----- YABS - Yet Another Backup Script ($script_ver $script_date) -------------------
# Filename:  host.conf
#
# Description:  This file is used by SSH and SCP to make secure noninteractive
# connections to a remote server.
#
# CONFIGURING SSH CONNECTIONS:
# Use the "ssh-keygen" utility to generate an rsa public/private key pair.
# Name the key something like "id_rsa.client_name" so it is easily
# identifiable.  Do not give the key a pass phrase.  The ssh connections all
# require a public/private key.  The ssh client's public RSA key must be copied
# to the ~/.ssh/authorized_keys file on the destination server.  This key must
# not require a passphrase as that would require entering a password, which
# can't be done in a cron job.  The corresponding private key must be on the
# machine where yabs.pl runs in the location specified in this host.conf
# file.  Directory and file permissions are important.  The RSA private key
# file must only be readable by the owner of the file, ie. "chmod 400
# filename".  The directory must only be writeable by the owner, ie. 700.  Use
# chmod to set permissions accordingly.
# 

# Each entry in host.conf needs to be verified to prove automatic ssh
# connections can be made.  As an example, to test the "webpage" host, use this
# command:
# 
# ssh -F host.conf webpage
# 
# The above command verifies the settings in the host.conf file for host
# "webpage".  You will be warned that the ssh server's key doesn't exist and
# you will be asked if you want to add it.  Answer yes.  Once this key is in
# the known_hosts file you won't be asked for it anymore.  Type exit to return
# to the ssh client shell.  Repeat the above configuration for all the host
# definitions in host.conf.
# 
# Copyright: 2003-2010 by John R Larsen  -  john\@larsen-family.us 
# Released under the same Artistic license as perl.
#
#-----------------------------------------------------------------------------
Host webpage
HostName = fully_qualified_domain_name
Port = port_number_on_server
UserKnownHostsFile = $working_dir/known_hosts
User = username_on_server
IdentityFile = $working_dir/id_rsa.private
#-----------------------------------------------------------------------------
Host put_host_name_here
HostName = fully_qualified_domain_name
Port = port_number_on_server
UserKnownHostsFile = $working_dir/known_hosts
User = username_on_server
IdentityFile = $working_dir/id_rsa.private
#-----------------------------------------------------------------------------
EOF
   close HOST_CONF;
   print "Created host.conf file.  You must edit this file with your settings.\n"
} # make_host_conf


#------------------------------------------------------------------------------
# process_cmd_line:  Function that uses GetOptions to process the command line
sub process_cmd_line 
{
# Process a command line full of arguments
   my $func_name = "process_cmd_line";
   # Reconfigure Getopt for no pass through so that any command line errors 
   # get caught.
   Getopt::Long::config ("no_pass_through");
   my $restart = FALSE;
   $rc = GetOptions ( 
      'd=o' => sub {$D = $_[1]; &set_log_dest},
      'bd=i' => \&baseline_day,
      'dd=i' => \&delta_days,
      'e=s' => \$email_address,
      'f=s' => \$yabs_dirlist,
      'fs=s' => \$fail_safe,
      'kd=i' => \&keep_days,
      'mb' => sub {&make_yabs_dirlist; unlink "$working_dir/$yabs_lock"; exit 0},
      'mdf' => sub {&make_yabs_df; unlink "$working_dir/$yabs_lock"; exit 0},
      'r' => sub {$fix_files = TRUE},
      't=s' => \&backup_type,
      'test' => sub {$test_mode = TRUE; &set_log_dest},
      '<>' => \&process_arg
   );
    
   # Die if GetOptions reports an error
   if (! $rc) {
      print STDERR "ERROR: Incorrect command line.  Unable to continue.\n";
      unlink "$working_dir/$yabs_lock";
      exit 1;
   }

   # Verify that $baseline_day is within valid range
   sub baseline_day {
      $baseline_day = $_[1];
      if (($baseline_day < 0) || ($baseline_day > 31)){
         print "ERROR: -bd value $baseline_day outside valid range of 0 to 31\n";
         unlink "$working_dir/$yabs_lock";
         exit 1;
      }
      # This needs to have a leading zero for single digit days
      $baseline_day = sprintf ("%02d", $baseline_day);
   }

   # Verify that $backup_type is supported
   sub backup_type {
      $backup_type = $_[1];
      if (!(($backup_type eq "daily") ||
         ($backup_type eq "delta") ||
         ($backup_type eq "rsync"))){
         print "ERROR: -t value \"$backup_type\" unsupported\n";
         unlink "$working_dir/$yabs_lock";
         exit 1;
      }
   }

   # Verify that $delta_days is within valid range
   sub delta_days {
      $delta_days = $_[1];
      if ($delta_days < 1){
         print "ERROR: -dd value $delta_days less than 1\n";
         unlink "$working_dir/$yabs_lock";
         exit 1;
      }
   }

   # Verify that $keep_days is within valid range
   sub keep_days {
      $keep_days = $_[1];
      if ($keep_days < 0){
         print "ERROR: -kd value $keep_days less than 0\n";
         unlink "$working_dir/$yabs_lock";
         exit 1;
      }
   }

   # Non option arguments processed here.  
   sub process_arg {
      print "ERROR: Non option args found: $_[0]\n";
      unlink "$working_dir/$yabs_lock";
      exit 1;
   }
} # process_cmd_line


#------------------------------------------------------------------
# purge_archives:  Function that deletes archives older than N days old.
# Arg0: DEST_DIR
# Arg1: backup type
# Arg2: Number of days to keep
# Arg3: Archive base name
sub purge_archives 
{
	my $func_name = "purge_archives";
   my $dest_dir = $_[0] ? $_[0] : die "ERROR: parameters missing in call to purge_archives: $!";
   my $backup_type = $_[1];
   my $keep_days = $_[2];
   my $base_name = $_[3];
   my $host_name = "";
   my $command;
   my $output;
   
   if ( $D & 0x1 ) { 
      printf ("[%d]$func_name:$$> Inside purge_archives\n", __LINE__);
      printf ("     dest_dir:    $dest_dir\n");
      printf ("     keep_days:   $keep_days\n");
      printf ("     backup_type: $backup_type\n");
      printf ("     base_name:   $base_name\n");
   }

   # Get out if $keep_days is 0 because that means keep all backups
   return 0 if ($keep_days eq 0);

   # Get out if $backup_type is "daily" because there is only one backup file
   return 0 if ($backup_type eq "daily");

   # For rsync need to isolate the remote host name if any
   if ($backup_type eq "rsync"){
      if ($dest_dir =~ m/(.*):(.*)/){
         # Getting here means the backups are on a remote host
         $host_name = $1;
         $dest_dir = $2;
      }
   }

   # Need to determine the oldest backup date to keep
   # Use "time" and "localtime" to figure out yy, mm, and dd for filenames
   my $sec; 
   my $min; 
   my $hour; 
   my $mday; 
   my $mon; 
   my $year; 
   my $wday; 
   my $yday; 
   my $isdst;
   ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time-(60*60*24*$keep_days));

   # Start with the calculated day and work backwards through 2008
   my $archive_name;
   my $yy;
   my $yyyy;
   my $mm;
   my $dd;
   while ($year >= 106) {
      $yy = sprintf("%02d", $year-100);
      $yyyy = sprintf("%02d", $year+1900);
      while ($mon >= 0) {
         $mm = sprintf("%02d", $mon+1);
         while ($mday >= 1) {
            $dd = sprintf("%02d", $mday);

            # Delete archives a day at a time
            if ($backup_type eq "rsync"){
               $archive_name = "$dest_dir" . "." . "$yy" . "$mm" . "$dd" . "*";
               if ($host_name eq ""){
                  # Getting here means the backups are on a local drive
                  if ( $D & 0x400 ) { 
                     printf ("[%d]$func_name:$$> Deleting archive: $archive_name\n", __LINE__);
                  }
                  $command = "rm -fr $archive_name";
                  $output = `$command 2>&1`;
                  print "$output";
               }
               else {
                  # Getting here means the backups are on a remote host
                  if ( $D & 0x400 ) { 
                     printf ("[%d]$func_name:$$> Deleting archive: $archive_name\n", __LINE__);
                  }
                  $command = "$SSH_PGM -F $working_dir/host.conf " . "$host_name " . "rm -fr " . "\"$archive_name\"";
                  $output = `$command 2>&1`;
                  print "$output";
               }
            }
            else {
               # delta archives are handled here
               $archive_name = "$dest_dir\/" . "$base_name" . "__" . "$yy" . "$mm" . "$dd" . "*" . ".zip";
               if ( $D & 0x400 ) { 
                  printf ("[%d]$func_name:$$> Deleting archive: $archive_name\n", __LINE__);
               }
               unlink glob("$archive_name");
            }
            $mday--;
         } # while days of the month

         # Delete archives a month at a time
         $mon--;
         $mm = sprintf("%02d", $mon+1);
         next if $mon < 0;
         if ($backup_type eq "rsync"){
            $archive_name = "$dest_dir" . "." . "$yy" . "$mm" . "*";
            if ($host_name eq ""){
               # Getting here means the backups are on a local drive
               if ( $D & 0x400 ) { 
                  printf ("[%d]$func_name:$$> Deleting archive: $archive_name\n", __LINE__);
               }
               $command = "rm -fr $archive_name";
               $output = `$command 2>&1`;
               print "$output";
            }
            else {
               # Getting here means the backups are on a remote host
               if ( $D & 0x400 ) { 
                  printf ("[%d]$func_name:$$> Deleting archive: $archive_name\n", __LINE__);
               }
               $command = "$SSH_PGM -F $working_dir/host.conf " . "$host_name " . "rm -fr " . "\"$archive_name\"";
               $output = `$command 2>&1`;
               print "$output";
            }
         }
         else {
            # delta archives are handled here
            $archive_name = "$dest_dir\/" . "$base_name" . "__" . "$yy" . "$mm" . "*" . ".zip";
            if ( $D & 0x400 ) { 
               printf ("[%d]$func_name:$$> Deleting archive: $archive_name\n", __LINE__);
            }
            unlink glob("$archive_name");
         }
      } # while months of the year

      # Delete archives a year at a time
      $year--;
      $yy = sprintf("%02d", $year-100);
      if ($backup_type eq "rsync"){
         $archive_name = "$dest_dir" . "." . "$yy" . "*";
         if ($host_name eq ""){
            # Getting here means the backups are on a local drive
            if ( $D & 0x400 ) { 
               printf ("[%d]$func_name:$$> Deleting archive: $archive_name\n", __LINE__);
            }
            $command = "rm -fr $archive_name";
            $output = `$command 2>&1`;
            print "$output";
         }
         else {
            # Getting here means the backups are on a remote host
            if ( $D & 0x400 ) { 
               printf ("[%d]$func_name:$$> Deleting archive: $archive_name\n", __LINE__);
            }
            $command = "$SSH_PGM -F $working_dir/host.conf " . "$host_name " . "rm -fr " . "\"$archive_name\"";
            $output = `$command 2>&1`;
            print "$output";
         }
      }
      else {
         # delta archives are handled here
         $archive_name = "$dest_dir\/" . "$base_name" . "__" . "$yy" . "*" . ".zip";
         if ( $D & 0x400 ) { 
            printf ("[%d]$func_name:$$> Deleting archive: $archive_name\n", __LINE__);
         }
         unlink glob("$archive_name");
      }
   } # while years
} # purge_archives


#------------------------------------------------------------------------------
# read_yabs_dirlist: Function to read options from yabs.dirlist file
sub read_yabs_dirlist 
{
   my $func_name = "read_yabs_dirlist";
   my $post_cmds_found = FALSE;
   my $line_num = 0;
   if ($D & 0x1) { printf ("[%d]$func_name:$$> Inside read_yabs_dirlist\n", __LINE__) }
   if ( -e "$yabs_dirlist" ) {
      open YABS_DIRLIST, "<", "$yabs_dirlist" or die "Cannot open $yabs_dirlist for reading: $!";
      while (<YABS_DIRLIST>) {
         $line_num++;
         if ($D & 0x2) { print "$_" }
         if (/^# NOT CONFIGURED/) {
            print STDERR "ERROR: You must edit $yabs_dirlist! Line $line_num still says NOT CONFIGURED\n";
            unlink "$working_dir/$yabs_lock";
            exit 1;
         }

         if (/^EMAIL_ADDRESS\s*(\S*)/) {
            # If the value isn't "none" then it was on the command line and don't change it
            if ($email_address eq "none"){
               $email_address = $1;
            }
            next;
         }

         if (/^BACKUP_TYPE\s*(\S*)/) {
            # If the value isn't "none" then it was on the command line and don't change it
            if ($backup_type eq "none"){
               $backup_type = $1;
               if (!(($backup_type eq "daily") ||
                  ($backup_type eq "delta") ||
                  ($backup_type eq "rsync"))){
                  unlink "$working_dir/$yabs_lock";
                  die "ERROR: Unsupported backup type found in $yabs_dirlist: $backup_type\n";
               }
            }
            next;
         }

         if (/^KEEP_DAYS\s*(\S*)/) {
            # If the value isn't zero (0) then it was on the command line and don't change it
            if ($keep_days eq 0){
               $keep_days = $1;
               if ($keep_days < 0){
                  print "ERROR: KEEP_DAYS value $keep_days in $yabs_dirlist less than 0\n";
                  unlink "$working_dir/$yabs_lock";
                  exit 1;
               }
            }
            next;
         }

         if (/^BASELINE_DAY\s*(\S*)/) {
            # If the value isn't 01 then it was on the command line and don't change it
            if ($baseline_day eq "01"){
               $baseline_day = sprintf("%02d", $1);
               if (($baseline_day < 0) || ($baseline_day > 31)){
                  print "ERROR: BASELINE_DAY value $baseline_day in $yabs_dirlist outside valid range of 0 to 31\n";
                  unlink "$working_dir/$yabs_lock";
                  exit 1;
               }
            }
            next;
         }

         if (/^DELTA_DAYS\s*(\S*)/) {
            # If the value isn't 1 then it was on the command line and don't change it
            if ($delta_days ==  1){
               $delta_days = $1;
               if ($delta_days < 1){
                  print "ERROR: DELTA_DAYS value $delta_days in $yabs_dirlist less than 1\n";
                  unlink "$working_dir/$yabs_lock";
                  exit 1;
               }
            }
            next;
         }

         if (/^WEBPAGE_REMOTE_DIR\s*(\S*)/) {
            $webpage_remote_dir = $1;
            next;
         }

         if (/^WEBPAGE_LOCAL_DIR\s*(\S*)/) {
            $webpage_local_dir = $1;
            next;
         }

         if (/^COMPRESS_RSYNC_BACKUP\s*(\S*)/) {
            $compress_rsync_backup = $1;
            next;
         }

         if (/^# RSYNC OPTIONS/) {
            # Found the beginning of the rsync options section
            while (<YABS_DIRLIST>){
               $line_num++;
               if ($D & 0x2) { print "$_" }
               if (/^# END RSYNC OPTIONS/) {
                  last;
               }

               # Skip comment lines
               if (/^#/) {
                  next;
               }
               
               # Append the option to the rsync options acquired thus far
               /(.*)/;
               $rsync_opts = $rsync_opts . $1 . " ";
            }
            next;
         }

         if (/^# ERROR REPORTING/) {
            # Found the beginning of the error reporting search strings
            $search_err_index = 0;
            while (<YABS_DIRLIST>){
               $line_num++;
               if ($D & 0x2) { print "$_" }
               if (/^# END ERROR REPORTING/) {
                  last;
               }

               # Skip comment lines
               if (/^#/) {
                  next;
               }
               
               # Add the search string to the search_err array
               /(.*)/;
               $search_err[$search_err_index++] = $1;
               if ( $D & 0x100 ) { 
                  printf ("[%d]$func_name:$$> Found error search term: $1\n", __LINE__);
               }
            }
            next;
         }

         # Do some error checking on yabs entries. Make sure that all four lines are present
         # and in the correct order.  If PRE_CMDS: found then look for other three lines.
         # Also look for trailing spaces after directory lines.  Also verify starting directories
         # exist and that local destination directories exist.  Also verify that host: only
         # appears in DEST_DIR: lines if the $backup_type is "rsync".
         if (/(^PRE_CMDS:.*)/) {
            $post_cmds_found = FALSE;
            my $pre_cmds = $1;
            while (<YABS_DIRLIST>){
               $line_num++;
               if ($D & 0x2) { print "$_" }
               # Allow comment lines inbetween
               if (/^#/) {
                  next;
               }

               # Expecting START_DIR: as only non comment line
               if (/^(START_DIR:.*)/) {
                  my $start_dir_line = $1;
                  # Check for trailing white space at end of $start_dir_line
                  if ($start_dir_line =~ m/\s\z/) {
                     unlink "$working_dir/$yabs_lock";
                     die "ERROR: $yabs_dirlist white space at end of line $line_num: \"$start_dir_line\"\n";
                  }
                  # Verify that the starting directory actually exists
                  $start_dir_line =~ m/^START_DIR:\s*(.*)/;
                  -d "$1" or print "WARNING: START_DIR doesn't exist at line $line_num of $yabs_dirlist: \"$start_dir_line\"\n";

                  # Now look for DEST_DIR:
                  while (<YABS_DIRLIST>){
                     $line_num++;
                     if ($D & 0x2) { print "$_" }
                     # Allow comment lines inbetween
                     if (/^#/) {
                        next;
                     }

                     # Expecting DEST_DIR: as only non comment line
                     if (/^DEST_DIR:\s*(.*)/) {
                        my $dest_dir_line = $1;
                        # Check for trailing white space at end of $dest_dir_line
                        if ($dest_dir_line =~ m/\s\z/) {
                           unlink "$working_dir/$yabs_lock";
                           die "ERROR: $yabs_dirlist white space at end of line $line_num: \"$dest_dir_line\"\n";
                        }
                        
                        # $backup_type should have been found by now. Only "rsync" backups can have
                        # a "host:" in the DEST_DIR: path. Bail out if there is a mismatch.
                        if ($dest_dir_line =~ m/(.*):.*/){
                           my $host_name = $1;
                           if ($backup_type ne "rsync"){
                              print "ERROR: backup type \"$backup_type\" does not support remote host in file \"$yabs_dirlist\"\n";
                              print "See line $line_num: \"DEST_DIR: $dest_dir_line\"\n";
                              unlink "$working_dir/$yabs_lock";
                              exit 1;
                           }

                           # Check for "Host $host_name:" in host.conf and bail out if it doesn't exist
                           my $host_found = FALSE;
                           open HOST_CONF, "<", "host.conf" or die "Cannot open host.conf for reading: $!";
                           while (<HOST_CONF>) {
                              if (/^Host\s*$host_name/) {
                                 $host_found = TRUE;
                                 last;
                              }
                           }
                           close HOST_CONF;
                           if ($host_found == FALSE) {
                              print "ERROR: remote host $host_name not found in host.conf. See line $line_num in $yabs_dirlist\n";
                              unlink "$working_dir/$yabs_lock";
                              exit 1;
                           }
                        }
                        else {
                           # Destination is local so verify that it actually exists
                           -d "$dest_dir_line" or print "WARNING: DEST_DIR doesn't exist at line $line_num of $yabs_dirlist: \"DEST_DIR: $dest_dir_line\"\n";
                        }

                        # Now look for POST_CMDS:
                        while (<YABS_DIRLIST>){
                           $line_num++;
                           if ($D & 0x2) { print "$_" }
                           # Allow comment lines inbetween
                           if (/^#/) {
                              next;
                           }

                           # Expecting POST_CMDS: as only non comment line
                           if (/^POST_CMDS:/) {
                              $post_cmds_found = TRUE;
                              last;
                           }
                           else {
                              unlink "$working_dir/$yabs_lock";
                              die "ERROR: $yabs_dirlist missing POST_CMDS: after line: \"DEST_DIR: $dest_dir_line\"\n";
                           }
                           last;
                        }
                     }
                     else {
                        unlink "$working_dir/$yabs_lock";
                        die "ERROR: $yabs_dirlist missing DEST_DIR: after line: \"$start_dir_line\"\n";
                     }
                     last;
                  }
               }
               else {
                  unlink "$working_dir/$yabs_lock";
                  die "ERROR: $yabs_dirlist missing START_DIR: after line: \"$pre_cmds\"\n";
               }
               last;
            }
            next;
         }
      }
      close YABS_DIRLIST;
      if (!(($backup_type eq "daily") ||
         ($backup_type eq "delta") ||
         ($backup_type eq "rsync"))){
         unlink "$working_dir/$yabs_lock";
         die "ERROR: Unsupported backup type found in $yabs_dirlist: $backup_type\n";
      }
      if ($post_cmds_found == FALSE){
         unlink "$working_dir/$yabs_lock";
         die "ERROR: Missing POST_CMDS: line after last DEST_DIR: line in file $yabs_dirlist\n";
      }
   }
   else {
      unlink "$working_dir/$yabs_lock";
      die "ERROR: $yabs_dirlist doesn't exist\n";
   }
} # read_yabs_dirlist


#------------------------------------------------------------------------------
# recurse_tree: Function that recurses through the directory tree starting
# with directory passed in $1.
sub recurse_tree 
{
   my $local_start_dir = $_[0] ? $_[0] : die "ERROR: Starting directory missing: $!";
   my $command;
   my $output;
   my $func_name = "recurse_tree";
   if ($D & 0x1) { printf ("[%d]$func_name:$$> Inside recurse_tree: $local_start_dir\n", __LINE__) }

   # Try to chdir into starting directory. If permission is denied then simply return.
   chdir "$local_start_dir" or return 0;

	# Display recurse level and pwd if debug turned on
	if ($D & 0x20) {
		printf ("[%d]$func_name:$$> RECURSE_LEVEL: $recurse_level in $local_start_dir\n", __LINE__);
	}

	# Show contents of directory if debug is turned on
	if ($D & 0x40) {
		printf ("[%d]$func_name:$$> ----- Contents of: $local_start_dir\n", __LINE__);
		my @contents = `$LS_PGM -AF "$local_start_dir"`;
      for (@contents){
         chomp $_;
         print "$_  ";
      }
      print "\n";
	}

   # If -r on command line then rename any "backup.*" files to "yabs.*"
   if($fix_files == TRUE){
      &fix_files($local_start_dir);
   }

	# If file yabs.no_recurse exists in the current directory and $recurse_level isn't zero
	# then return.  $recurse_level equal to zero means this is the top level directory.  Not
	# checking for yabs.no_recurse file allows having an entry in yabs.dirlist that
   # starts recursing in an arbitrary directory even if yabs.no_recurse is there.  This
	# makes it possible to have a directory marked yabs.no_recurse for a higher level
	# starting point but still use it as a starting directory for a subsequent entry in
	# a yabs.dirlist file.
	if ((-e "yabs.no_recurse") && ($recurse_level != 0)){
		if ($test_mode == TRUE){
			print "yabs.no_recurse file found in: $local_start_dir\n";
		}
		return 0;
	}

	# Check if recurse level has hit the fail safe value
	if ($recurse_level > $fail_safe){
		printf ("[%d]$func_name:$$> ERROR: Hit fail safe level: $fail_safe in \"$local_start_dir\"\n", __LINE__);
		return 1;
	}

	# If file yabs.enter_cmds exists in the current directory then source it
	if (-e "yabs.enter_cmds"){
      # Make sure the execute bit is set
      chmod 0755, "yabs.enter_cmds";

		if ($test_mode == TRUE){
			print "yabs.enter_cmds file found in: $local_start_dir\n";
		}
      my $output = `yabs.enter_cmds 2>&1`;
      print "$output";
	}

	# Check if file "yabs.list" exists
	if (-e "yabs.list"){
		if ($test_mode == TRUE){
			print "yabs.list file found in:       $local_start_dir\n";
		}

		# Add an entry to the email file
		if ($test_mode == TRUE){
         print "----- TEST MODE ------------------------------------------------\n";
      }
      else {
         print "----------------------------------------------------------------\n";
      }
      my $start_time = &get_time;
      print "$backup_type backup started at: $start_time\n";
		print "Source directory:        $local_start_dir\n";
		print "Destination directory:   $dest_dir\n";

		# Remove the first slash in the path, replace all the rest with dots, and replace spaces with circumflex ^
		$command = "pwd | sed  -e 's/\\///' -e 's/\\//./g' -e 's/ /^/g'";
		chomp (my $filename = `$command`);
      &purge_archives ($dest_dir, $backup_type, $keep_days, $filename);
		my $old_size_date_name = "";
      my $size_date_name;
      # Check backup type
      if ($backup_type eq "daily"){
         $filename = $filename . ".zip";
         # Capture info about existing backup file for later comparison
         if (-e "$dest_dir/$filename"){
            $command = "$LS_PGM -lo \"$dest_dir/$filename\" | sed  -e 's/[^ ]* *[^ ]* *[^ ]*//'";
            $old_size_date_name=`$command`;
         }
      }
      elsif ($backup_type eq "delta"){
         if (("$day" eq "$baseline_day") || ("$baseline_day" eq "00")){
            # Put "-fb" in filename to indicate "full backup"
            $filename = $filename . "__" . $backup_date . "-fb" . ".zip";
         }
         else {
            # Put "-pb$delta_days" to indicate "partial backup" with number of days in the partial backup
            $filename = $filename . "__" . $backup_date . "-pb$delta_days" . ".zip";
         }
         print "Created: \n";
      }
      else {
         # Unknown backup type!
            unlink "$working_dir/$yabs_lock";
            die "ERROR:  Unknown backup mode detected: $backup_type\n";
      }

		if ($test_mode == TRUE){
         $size_date_name ="   test mode:  $dest_dir/$filename";
      }
		else {
         # Update the webpage before each backup so incremental progress can be detected during long backups
			&webpage;

			# Create the zipfile using yabs.list to determine files included.  Note that zip doesn't care if
			# yabs.list is in unix or dos format, so there is no need to remove ^M from yabs.list.
         if ("$backup_type" eq "delta"){
            if (("$day" eq "$baseline_day") || ("$baseline_day" eq "00")){
               # Make a full backup on the $baseline day specified by -bd. The default is the first of the month.
               # If $baseline_day has the value "00" then do a full backup.
               $command = "cat yabs.list | $ZIP_PGM -urqy \"$dest_dir/$filename\" -@";
               $output = `$command 2>&1`;
               print "$output\n";
               # The next line removes permissions field, the numeric field, and the owner field
               $command = "$LS_PGM -lo \"$dest_dir/$filename\" | sed  -e 's/[^ ]* *[^ ]* *[^ ]*//'";
               $size_date_name = `$command`;
            }
            else {
               # Make a delta backup on the other days of the month
               # Use "time" and "localtime" to figure out yyyymmdd for $delta_days amount in the past
               my $sec; 
               my $min; 
               my $hour; 
               my $zip_mday; 
               my $zip_mon; 
               my $zip_year; 
               my $wday; 
               my $yday; 
               my $isdst;
               ($sec,$min,$hour,$zip_mday,$zip_mon,$zip_year,$wday,$yday,$isdst) = localtime(time-(60*60*24*$delta_days));
               $zip_year += 1900;
               $zip_mday = sprintf("%02d", $zip_mday);
               $zip_mon = sprintf("%02d", $zip_mon+1);
               $command = "cat yabs.list | $ZIP_PGM -urqy -t $zip_mon$zip_mday$zip_year \"$dest_dir/$filename\" -@";
               $output = `$command 2>&1`;
               print "$output\n";
               if (-e "$dest_dir/$filename"){
                  # The next line removes permissions field, the numeric field, and the owner field
                  print "Created:\n";
                  $command = "$LS_PGM -lo \"$dest_dir/$filename\" | sed  -e 's/[^ ]* *[^ ]* *[^ ]*//'";
                  $size_date_name = `$command`;
               }
               else {
                  $size_date_name = "   No backup file created";
               }
            }
         }
         else {
            # daily mode
            $command = "cat yabs.list | $ZIP_PGM -urqy \"$dest_dir/$filename\" -@";
            $output = `$command 2>&1`;
            print "$output\n";
            $command = "$LS_PGM -lo \"$dest_dir/$filename\" | sed  -e 's/[^ ]* *[^ ]* *[^ ]*//'";
            $size_date_name = `$command`;
            # Determine if the archive was updated or not and output to email accordingly
            if ("$old_size_date_name" eq "$size_date_name"){
               print "Archive not changed:\n";
            }
            else {
               print "Updated:\n";
            }
         }
		}

		print  "$size_date_name\n";
      my $end_time = &get_time;
		print "yabs ended at:   $end_time\n";

		# If file yabs.exit_cmds exists in the directory then source it
		if (-e "yabs.exit_cmds"){
         # Make sure the execute bit is set
         chmod 0755, "yabs.exit_cmds";

			if ($test_mode == TRUE){
				print "yabs.exit_cmds file found in:  $local_start_dir\n";
         }
         $command = "yabs.exit_cmds \"$dest_dir\" \"$filename\"";
         $output = `$command`;
         print $output;
		}
		return 0;
	}

	# Getting here means yabs.list wasn't found and need to recurse all subdirectories
   # in this directory looking for instances of yabs.list
   my @list = `$LS_PGM "$local_start_dir"`;
   for (@list){
      chomp $_;
      if (-d "$_"){
         $recurse_level++;
         my $new_start_dir = $local_start_dir . "/" . $_;
         &recurse_tree ($new_start_dir);
         # Need to chdir back to the correct directory
         chdir "$local_start_dir";
         $recurse_level--;
      }
   }

	# If file yabs.exit_cmds exists in the directory then source it.  No backup operation
	# has been done at this point, so no arguments are passed to yabs.exit_cmds.
	if (-e "yabs.exit_cmds"){

		if ($test_mode == TRUE){
			print "yabs.exit_cmds file found in:  $local_start_dir\n";
		}
      $command = "yabs.exit_cmds";
      $output = `$command 2>&1`;
      print $output;
	}
} # recurse_tree


#------------------------------------------------------------------------------
# recurse_tree_rsync: Function that recurses through the directory tree starting
# with directory passed in $1.  This function is only used for rsync backups.
sub recurse_tree_rsync 
{
   my $local_start_dir = $_[0] ? $_[0] : die "ERROR: Starting directory missing: $!";
   my $local_rsync_opts = $rsync_opts;
   my $func_name = "recurse_tree_rsync";
   if ($D & 0x1) { printf ("[%d]$func_name:$$> Inside recurse_tree_rsync: $local_start_dir\n", __LINE__) }

   # Check if the debug_flags file has changed
   #&set_debug_output;

   # Try to chdir into starting directory. If permission is denied then simply return.
   chdir "$local_start_dir" or return 0;

	# Display recurse level and pwd if debug turned on
	if ($D & 0x20) {
		printf ("[%d]$func_name:$$> RECURSE_LEVEL: $recurse_level in $local_start_dir\n", __LINE__);
	}

	# Show contents of directory if debug is turned on
	if ($D & 0x40) {
		printf ("[%d]$func_name:$$> ----- Contents of: $local_start_dir\n", __LINE__);
		my @contents = `$LS_PGM -AF "$local_start_dir"`;
      for (@contents){
         chomp $_;
         print "$_    ";
      }
      print "\n";
	}

   # If -r on command line then rename any "backup.*" files to "yabs.*"
   if($fix_files == TRUE){
      &fix_files($local_start_dir);
   }

	# If file yabs.no_recurse exists in the current directory and $recurse_level isn't zero
	# then return.  $recurse_level equal to zero means this is the top level directory.  Not
	# checking for yabs.no_recurse file allows having an entry in yabs.dirlist that
   # starts recursing in an arbitrary directory even if yabs.no_recurse is there.  This
	# makes it possible to have a directory marked yabs.no_recurse for a higher level
	# starting point but still use it as a starting directory for a subsequent entry in
	# a yabs.dirlist file.
	if ((-e "yabs.no_recurse") && ($recurse_level != 0)){
		if ($test_mode == TRUE){
			print "yabs.no_recurse file found in: $local_start_dir\n";
		}
		return 0;
	}

	# Check if recurse level has hit the fail safe value
	if ($recurse_level > $fail_safe){
		printf ("[%d]$func_name:$$> ERROR: Hit fail safe level: $fail_safe in \"$local_start_dir\"\n", __LINE__);
		return 1;
	}

	# If file yabs.enter_cmds exists in the current directory then source it
	if (-e "yabs.enter_cmds"){
      # Make sure the execute bit is set
      chmod 0755, "yabs.enter_cmds";

		if ($test_mode == TRUE){
			print "yabs.enter_cmds file found in: $local_start_dir\n";
		}
      my $output = `yabs.enter_cmds 2>&1`;
      print "$output";
	}

	# Check if file "yabs.rsync" exists
	if (-e "yabs.rsync"){
		if ($test_mode == TRUE){
			print "yabs.rsync file found in: $local_start_dir\n";
		}

      # If this backup is to a remote host then the ssh option needs to be added to the rsync options
      if ($dest_dir =~ m/.*:.*/){
         $local_rsync_opts = $local_rsync_opts . "-e \"$SSH_PGM -F $working_dir/host.conf\"" . " ";
      }

      # If --backup is in the $rsync_opts then the option --backup-dir=dir_name needs to be added
      # to $local_rsync_opts to specify where the files are put.  The $dest_dir has a "host:" value
      # for remote backups that needs to be removed.
      my $yabs_dir = "";
      if ($rsync_opts =~ m/.*--backup.*/) {
         # Check if $dest_dir has a colon in it.
         if ($dest_dir =~ m/.*:(.*)/){
            $yabs_dir = $1;
         }
         else {
            $yabs_dir = $dest_dir;
         }
         $local_rsync_opts = $local_rsync_opts . "--backup-dir=\"${yabs_dir}.$backup_yymmddhhmm\"" . " ";
      }

      # If -test is on the yabs.pl command line then need to add --dry-run to the rsync options
      if ($test_mode == TRUE){
         $local_rsync_opts = $local_rsync_opts . "--dry-run" . " ";
      }
      
		# Add an entry to the email file
		if ($test_mode == TRUE){
         print "----- TEST MODE ------------------------------------------------\n";
      }
      else {
         print "----------------------------------------------------------------\n";
      }
      my $start_time = &get_time;
      print "rsync backup started at: $start_time\n";
		print "Source directory:        $local_start_dir\n";
		print "Destination directory:   $dest_dir$local_start_dir\n";
		print "rsync options:           $local_rsync_opts\n";

      # Update the webpage before each backup so incremental progress can be detected during long backups
      &webpage;

      # Display command line if debug is enabled
      if (($D & 0x80) || ($test_mode == TRUE)){
         print "-------------------------------------------------\n";
         print "This is the rsync command used for this backup:\n";
         print "$RSYNC_PGM\n";
         print "$local_rsync_opts\n";
         print "$local_start_dir\n";
         print "$dest_dir\n";
         print "-------------------------------------------------\n";
      }
      # Now assemble the entire rsync command line and execute it
      my $command = $RSYNC_PGM . " " . $local_rsync_opts . "\"$local_start_dir\"" . " " . "\"$dest_dir\"";
      my $output = `$command 2>&1`;
      print $output;

      my $end_time = &get_time;
      print "rsync backup ended at: $end_time\n";

		return 0;
	}

	# Getting here means yabs.rsync wasn't found and need to recurse all subdirectories
   # in this directory looking for instances of yabs.rsync
   my @list = `$LS_PGM "$local_start_dir"`;
   for (@list){
      chomp $_;
      if (-d "$_"){
         $recurse_level++;
         my $new_start_dir = $local_start_dir . "/" . $_;
         &recurse_tree_rsync ($new_start_dir);
         # Need to chdir back to the correct directory
         chdir "$local_start_dir";
         $recurse_level--;
      }
   }

	# If file yabs.exit_cmds exists in the directory then source it.  No backup operation
	# has been done at this point, so no arguments are passed to yabs.exit_cmds.
	if (-e "yabs.exit_cmds"){

		if ($test_mode == TRUE){
			print "yabs.exit_cmds file found in:  $local_start_dir\n";
		}
      my $command = "yabs.exit_cmds";
      my $output = `$command 2>&1`;
      print $output;
	}
} # recurse_tree_rsync


#------------------------------------------------------------------------------
# fix_files:  Function that copies "backup.*" files to "yabs.*"
sub fix_files 
{
   my $current_dir = $_[0] ? $_[0] : die "ERROR: Current directory name missing: $!";

   my $func_name="set_debug_output";

   # yabs.pl used to be named backup.pl and all the control files used "backup" instead
   # of "yabs" in their names. Check for the old names here and change them if found.
   if(-e "backup.no_recurse"){
      copy "backup.no_recurse", "yabs.no_recurse";
      printf ("Copied backup.no_recurse to yabs.no_recurse in: $current_dir\n");
   }

   if(-e "backup.list"){
      copy "backup.list", "yabs.list";
      printf ("Copied backup.list to yabs.list in: $current_dir\n");
   }

   if(-e "backup.enter_cmds"){
      copy "backup.enter_cmds", "yabs.enter_cmds";
      printf ("Copied backup.enter_cmds to yabs.enter_cmds in: $current_dir\n");
   }

   if(-e "backup.exit_cmds"){
      copy "backup.exit_cmds", "yabs.exit_cmds";
      printf ("Copied backup.exit_cmds to yabs.exit_cmds in: $current_dir\n");
   }

   if(-e "backup.rsync"){
      copy "backup.rsync", "yabs.rsync";
      printf ("Copied backup.rsync to yabs.rsync in: $current_dir\n");
   }

   if(-e "backup.df"){
      copy "backup.df", "yabs.df";
      printf ("Copied backup.df to yabs.df in: $current_dir\n");
   }
} # fix_files


#------------------------------------------------------------------------------
# set_debug_output:  Function that checks contents of file debug_flags and
# updates $D if the value has changed.
sub set_debug_output 
{
   my $func_name="set_debug_output";

   # Check the existence of a file named "debug_flags" and use it if conditions are right
   if (-s "debug_flags") {
      open (DEBUG_FLAGS, "<", "debug_flags") 
         or die "Can't open debug_flags file: $!";

      # Read the value from the file, convert to a hex number and save for later use
      chomp ($debug_flags = <DEBUG_FLAGS>);
      close DEBUG_FLAGS;
      $debug_flags = oct $debug_flags;
   } else {
      $debug_flags="0";
   }

   # Make changes if contents of file debug_flags has changed since the last reading
   if ($debug_flags != $old_debug_flags) {
      $D = $debug_flags;
      # If MSB of debug_flags has changed then need to change log destination
      if (($old_debug_flags & 0x80000000) != ($debug_flags & 0x80000000)) {
         &set_log_dest;
      }
      # Save current flags value for later use
      $old_debug_flags = $debug_flags;
   }

   # This line is at the end because $D is set in this function
   if (($D & 0x1) != 0) {
      printf ("[%d]$func_name:$$> Inside set_debug_output (Flags: 0x%x)\n", __LINE__, $D);
   }
} # set_debug_output

#------------------------------------------------------------------------------
# set_log_dest:  Function that sets log output destination based on D$
# Arg1 - "clear" clears logfile else the logfile is opened for append
sub set_log_dest 
{
   my $clear = $_[0] ? $_[0] : "append";
   close LOG;
   if ($test_mode == TRUE) {
      # open the logfile with ">" to zero it out
      open LOG, ">", "logfile" or warn "Can't open logfile: $!";
      print LOG "Test mode. Nothing captured to logfile.\n";
      close LOG;
   }
   # The MSB of D$ determines where program output goes.  If the bit is clear (default)
   # then output goes to "logfile".  If it is set then output goes to STDOUT.  All
   # output is sent to STDOUT if running in test mode (ie. -test on command line).
   if (($D & 0x8000_0000) || ($test_mode == TRUE)) {
      # Getting here means the bit is set and output goes to STDOUT
      open LOG, ">-"
         or die "Can't redirect LOG output to STDOUT: $!";
      select LOG;
      $| = 1; #Flush log entries immediately
      select STDOUT;
   } else {
      # Getting here means the bit is clear and output goes to the logfile
      if ($clear eq "clear") {
         # open the logfile with ">" to zero it out
         open LOG, ">", "logfile" or warn "Can't open logfile: $!";
         close LOG;
      }
      # Open filehandle to logfile and set autoflush on for logfile
      open LOG, ">>", "logfile" or warn "Can't open logfile for append: $!";
      select LOG;
      $| = 1; #Flush log entries immediately
      sleep 1; # Give OS time to truncate logfile so that tailing the file works
   }
} # set_log_dest


#------------------------------------------------------------------------------
# set_working_dir: Function that sets the WORKING_DIR based on $0
sub set_working_dir 
{
   # Configure working directory
   $working_dir=`dirname $0`;
   chomp $working_dir;
   if ($working_dir eq ".") {
	   $working_dir=`pwd`;
      chomp $working_dir;
   }
   chdir "$working_dir";

} # set_working_dir


#-------------------------------------------------------------------------------
# setup_env.  Function that reads in a configuration file that sets up where
# all the host resident programs are located and various filter variables
# used by different OS.
sub setup_env 
{
   my $func_name="setup_env";
   if ( $D & 0x1 ) { printf ("[%d]$func_name:$$> Inside setup_env\n", __LINE__) }

   my $file = "yabs.conf";
   my $pgm_filter = '^(\S*)';    # Regex to strip off any trailing arguments to the program
   my $program;                  # Holds the program name less any arguments to the program
   my $return;

   # Check for existence of configuration file.  If it doesn't exist then create one.
   if ( -s $file ) {
      # Process configuration file
      unless ($return = do "./$file") {
             die "couldn't parse $file: $@" if $@;
             die "You must edit $file to select your operating system"    unless defined $return;
             die "couldn't run $file"       unless $return;
      }

      # Verify programs read in from configuration file are valid and executable or die
      # Verify $DF_PGM
      $DF_PGM =~ /not set/ and die "Check $file.  Value for \$DF_PGM needs to be set\n";
      $DF_PGM =~ /$pgm_filter/;
      $program = $1;
      -x $program or die "Check $file.  Value for \$DF_PGM: \"$program\" not executable: $!\n";

      # Verify $HOST_PGM
      $HOST_PGM =~ /not set/ and die "Check $file.  Value for \$HOST_PGM needs to be set\n";
      $HOST_PGM =~ /$pgm_filter/;
      $program = $1;
      -x $program or die "Check $file.  Value for \$HOST_PGM: \"$program\" not executable: $!\n";
      # Get the name of the host and remove the newline
      chomp ($host = `$HOST_PGM`);

      # Verify $LS_PGM
      $LS_PGM =~ /not set/ and die "Check $file.  Value for \$LS_PGM needs to be set\n";
      $LS_PGM =~ /$pgm_filter/;
      $program = $1;
      -x $program or die "Check $file.  Value for \$LS_PGM: \"$program\" not executable: $!\n";

      # Verify $MAIL_PGM
      $MAIL_PGM =~ /not set/ and die "Check $file.  Value for \$MAIL_PGM needs to be set\n";
      $MAIL_PGM =~ /$pgm_filter/;
      $program = $1;
      -x $program or die "Check $file.  Value for \$MAIL_PGM: \"$program\" not executable: $!\n";

      # Verify $PS_PGM
      $PS_PGM =~ /not set/ and die "Check $file.  Value for \$PS_PGM needs to be set\n";
      $PS_PGM =~ /$pgm_filter/;
      $program = $1;
      -x $program or die "Check $file.  Value for \$PS_PGM: \"$program\" not executable: $!\n";

      # Verify $RSYNC_PGM
      $RSYNC_PGM =~ /not set/ and die "Check $file.  Value for \$RSYNC_PGM needs to be set\n";
      $RSYNC_PGM =~ /$pgm_filter/;
      $program = $1;
      -x $program or warn "Check $file.  Value for \$RSYNC_PGM: \"$program\" not executable: $!\n";

      # Verify $SSH_PGM
      $SSH_PGM =~ /not set/ and die "Check $file.  Value for \$SSH_PGM needs to be set\n";
      $SSH_PGM =~ /$pgm_filter/;
      $program = $1;
      -x $program or warn "Check $file.  Value for \$SSH_PGM: \"$program\" not executable: $!\n";

      # Verify $SCP_PGM
      $SCP_PGM =~ /not set/ and die "Check $file.  Value for \$SCP_PGM needs to be set\n";
      $SCP_PGM =~ /$pgm_filter/;
      $program = $1;
      -x $program or die "Check $file.  Value for \$SCP_PGM: \"$program\" not executable: $!\n";

      # Verify $TELNET_PGM
      $TELNET_PGM =~ /not set/ and die "Check $file.  Value for \$TELNET_PGM needs to be set\n";
      $TELNET_PGM =~ /$pgm_filter/;
      $program = $1;
      -x $program or die "Check $file.  Value for \$TELNET_PGM: \"$program\" not executable: $!\n";

      # Verify $W_PGM
      $W_PGM =~ /not set/ and die "Check $file.  Value for \$W_PGM needs to be set\n";
      $W_PGM =~ /$pgm_filter/;
      $program = $1;
      -x $program or die "Check $file.  Value for \$W_PGM: \"$program\" not executable: $!\n";

      # Verify $ZIP_PGM
      $ZIP_PGM =~ /not set/ and die "Check $file.  Value for \$ZIP_PGM needs to be set\n";
      $ZIP_PGM =~ /$pgm_filter/;
      $program = $1;
      -x $program or die "Check $file.  Value for \$ZIP_PGM: \"$program\" not executable: $!\n";

      # Put certain variables into the environment that are required by child processes
      $ENV{SCP_PGM} = "$SCP_PGM";
      $ENV{HOST} = "$host";
      $ENV{DF_PGM} = "$DF_PGM";
      $ENV{BACKUP_YYMMDDHHMM} = "$backup_yymmddhhmm";

   } else {

      # Getting here means no yabs.conf file was found.  Create a new one.
      open YABS_CONF, ">", "$file" or die "Cannot open $file for writing: $!";
      print YABS_CONF << "EOF";
# NOT CONFIGURED - DELETE THIS LINE ONCE THIS FILE HAS BEEN CONFIGURED
#----- YABS - Yet Another Backup Script ($script_ver $script_date) -------------------
# Filename:  yabs.conf
# This is the program locations configuration file for the yabs program.
# Uncomment the lines that correspond to your operating system.  The program 
# locations are based on standard locations, but might need to be changed if 
# your installation is different.  Only change the path to the program. Don't 
# change any arguments on the program lines. They are required for yabs 
# operation.

# Solaris values
#our \$DF_PGM ='/usr/ucb/df';
#our \$HOST_PGM ='/usr/ucb/hostname';
#our \$LS_PGM ='/bin/ls';
#our \$MAIL_PGM ='/usr/ucb/mail';
#our \$PS_PGM ='/bin/ps';
#our \$RSYNC_PGM ='/usr/local/bin/rsync';
#our \$SCP_PGM ='/usr/local/bin/scp';
#our \$SSH_PGM ='/usr/local/bin/ssh';
#our \$TELNET_PGM ='/usr/bin/telnet';
#our \$W_PGM ='/usr/bin/w';
#our \$ZIP_PGM ='/usr/bin/zip';

# Linux 2.x values
#our \$DF_PGM = "/bin/df -h";
#our \$HOST_PGM ='/bin/hostname';
#our \$LS_PGM ='/bin/ls';
#our \$MAIL_PGM ='/usr/bin/mail';
#our \$PS_PGM ='/bin/ps';
#our \$RSYNC_PGM ='/usr/bin/rsync';
#our \$SCP_PGM ='/usr/bin/scp';
#our \$SSH_PGM ='/usr/bin/ssh';
#our \$TELNET_PGM ='/usr/bin/telnet';
#our \$W_PGM ='/usr/bin/w';
#our \$ZIP_PGM ='/usr/bin/zip';

# cygwin 1.5.x values assuming standard install to c:\\cygwin
# Note: You need to install the following non base cygwin packages:
#       email
#       inetutils
#       openssh
#       rsync
#       procps
#       zip
#
#our \$DF_PGM = "/usr/bin/df -h";
#our \$HOST_PGM ='/usr/bin/hostname';
#our \$LS_PGM ='/usr/bin/ls';
#our \$MAIL_PGM ='/usr/bin/email';
#our \$PS_PGM ='/usr/bin/procps';
#our \$RSYNC_PGM ='/usr/bin/rsync';
#our \$SCP_PGM ='/usr/bin/scp';
#our \$SSH_PGM ='/usr/bin/ssh';
#our \$TELNET_PGM ='/usr/bin/telnet';
#our \$W_PGM ='/usr/bin/w';
#our \$ZIP_PGM ='/usr/bin/zip';

# Copyright: 2003-2010 by John R Larsen  -  john\@larsen-family.us
# Released under the same Artistic license as perl
EOF
      close YABS_CONF;
      print STDERR "Created $file file.  You must edit this file to select your operating system.\n";

      # Create host.conf unless it already exists
      &make_host_conf;

      # Create yabs.dirlist unless it already exists. Don't overwrite an existing file.
      if (! (-e "yabs.dirlist")) {
         &make_yabs_dirlist;
      }
      unlink "$working_dir/$yabs_lock";
      exit 0;
   }
} # setup_env


#------------------------------------------------------------------
# test_port:  Function that tests if a connection can be made to a ssh port on a remote computer.
# It uses telnet to connect to the port.  It looks for "Connected to" in the response.
# Arg0: is the host name.      Fatal error if not set.
# Arg1: is the port to test.   Defaults to 22 if not set.
# Arg2: is the timeout value.  Defaults to 30 if not set.
# Returns:  GOOD_PORT if able to connect to port or BAD_PORT if connection is refused or times out
sub test_port 
{
	my $func_name = "test_port";
   my $pid;
   my $host = $_[0] ? $_[0] : die "host missing: $!";
   my $port = $_[1] ? $_[1] : 22;
   my $timeout = $_[2] ? $_[2] : 30;
   if ( $D & 0x1 ) { printf ("[%d]$func_name:$$> Inside test_port (host: $host   port: $port   timeout: $timeout)\n", __LINE__) }

   # Use fork so the test_port can be terminated if it takes too long
   TEST_PORT_FORK: {
      if ($pid = fork) {
         # Parent ends up here
         my $counter = 0;
         my $rc_pid;
         # Test child pid every second until the timeout value is reached
         while ($counter < $timeout) {
            if ($pid == (waitpid ($pid, WNOHANG))) {
               return GOOD_PORT;;
            }
            sleep 1;
            $counter++;
         }
         # Getting here means the port test timed out
         kill 9, $pid;
         printf ("[%d]$func_name:$$> test_port timed out (host: $host   port: $port   timeout: $timeout)\n", __LINE__);

         # Increment $bad_port_cntr
         $bad_port_cntr++;
         printf ("[%d]$func_name:$$> $bad_port_cntr bad test ports have occurred to $host port $port\n", __LINE__);
         return BAD_PORT;
      }
      elsif (defined $pid) { # $pid is zero here if defined
         # child ends up here
         # Change to the /tmp so the extraneous "1" file always is written there. The telnet test
         # ends up writing its output to a file named "1". I couldn't figure out how to prevent this.
         # The child process terminates so chdir doesn't affect operation of yabs.pl.
         chdir "/tmp";
         # A good telnet connection to an ssh server will have "Connected to" in the data
         my $data = `echo test | $TELNET_PGM $host $port 2>1`;
         if ( $D & 0x10 ) { printf ("[%d]$func_name:$$> Data from telnet connection:\n$data\n", __LINE__) }
         if ( $data =~ /.*Connected to.*/) {
            # Successful connection so exit normally
            # Solaris 9 didn't kill child process with a simple "exit 0".  The brute force
            # approach of having the child kill itself seems to work.
            kill 9, $$;
         }
         # If unable to connect then sleep longer than timeout so parent kills us
         sleep (10 + $timeout);
      }
      elsif ($! == EAGAIN) {
         # EAGAIN is the supposedly recoverable fork error
         sleep 5;
         redo TEST_PORT_FORK;
      }
      else {
         # weird fork error occurred
         die "Can't fork: $!\n";
      }
   }
} # test_port


#------------------------------------------------------------------------------
# webpage. Function that creates a web page and copies it to a webserver as 
# yabs progresses.  This function simply returns if $webpage_remote_dir and
# $webapge_local_dir are both "none".
sub webpage
{
   my $webserver_domain_name = "none";
   my $webserver_domain_port = "none";
   my $func_name="webpage";
   if ( $D & 0x1 ) { printf ("[%d]$func_name:$$> Inside webpage\n", __LINE__) }

   # Check if $webpage_remote_dir and $webpage_local_dir are both changed from the default "none"
   return 0 if ($webpage_remote_dir eq "none" && $webpage_local_dir eq "none");

   # Getting here means $webpage_remote_dir or $webpage_local_dir is a path instead of "none"

   WEB_PAGE_FORK: {
      if (my $pid = fork) {
         # Parent ends up here.  Simply return.
         return 0;
      }
      elsif (defined $pid) { # $pid is zero here if defined
         # child ends up here
         # Note: On Solaris 9 simply using "exit" in the child didn't work and processes
         # go restarted. Having the child use "kill 9, $$" on itself works.  This wasn't
         # an issue in Linux or cygwin.

         # Capture the creation time for the webpage
         my $date = &get_time;

         # A race condition on Cygwin on Win2000 can create an invisible file that can't be overwritten
         # or erased if the unlink command is used.  The work around is to simply open the file for
         # writing twice.  The first time with ">" to simply overwrite the previous contents
         # and then open it again with ">>" so it is appended.
         open WEBPAGE, ">", "$working_dir/webpage.html" or die "Cannot open $working_dir/webpage.html for writing: $!";

         # Redirect all output to file handle WEBPAGE
         my $test_mode_status = "";
         if ($test_mode == TRUE){
            $test_mode_status = "<h1>TEST MODE</h1>";
         }
         print WEBPAGE << "EOF";
         <HTML>
         <HEAD>
            <TITLE>$host:  $backup_type backup report for file $yabs_dirlist</TITLE>
         </HEAD>
         <BODY>
            $test_mode_status
            <h1>$host:  $backup_type backup report for file $yabs_dirlist</h1>
            <h2> Time: $date</h2>

EOF

         # Open it the second time with appending enabled
         open WEBPAGE, ">>", "$working_dir/webpage.html" or die "Cannot open $working_dir/webpage.html for writing: $!";

         # Copy the logfile into the webpage
         print WEBPAGE "<pre>\n";
         my $logfile = `cat $working_dir/logfile 2>&1`;
         print WEBPAGE "$logfile";
         print WEBPAGE "</pre>\n";

         print WEBPAGE "</BODY> </HTML>\n";

         # All done building the message so close the file handle
         close WEBPAGE;

         # Make a local copy if $webpage_local_dir is not "none"
         if ($webpage_local_dir ne "none") {
            copy "$working_dir/webpage.html", "$webpage_local_dir/$host.$yabs_dirlist.html";
         }

         # Only do the webserver update if $webpage_remote_dir isn't "none"
         kill 9, $$ if ($webpage_remote_dir eq "none");
         
         # Need to extract the webserver domain name and port from host.conf
         if ( -e "$working_dir/host.conf" ) {
            open HOST_CONF, "<", "$working_dir/host.conf" or die "Cannot open $working_dir/host.conf for reading: $!";
            while (<HOST_CONF>) {
               if (/^Host webpage/) {
                  # Found the beginning of the webserver host section
                  while (<HOST_CONF>){
                     if (/^HostName = (\S*)/) {
                        $webserver_domain_name = $1;
                        next;
                     }

                     if (/^Port = (\S*)/) {
                        $webserver_domain_port = $1;
                        next;
                     }

                     if (($webserver_domain_name ne "none") && ($webserver_domain_port ne "none")){
                        # Both terms found so stop looking for them
                        last;
                     }

                     # If start of another host definition is found then get out
                     if (/^Host /) {
                        last;
                     }
                  }
                  next;
               }
            }
            close HOST_CONF;
         }
         else {
            # Can't proceed if host.conf file is missing
            printf ("[%d]$func_name:$$> host.conf file missing\n", __LINE__);
            kill 9, $$;
         }

         # Only proceed if webserver's domain name and port were found
         if (($webserver_domain_name eq "none") || ($webserver_domain_port eq "none")){
            printf ("[%d]$func_name:$$> webserver domain name or port not found in host.conf\n", __LINE__);
            kill 9, $$;
         }

         # Only attempt an scp copy if the port checks out good
         my $port_state = &test_port ($webserver_domain_name, $webserver_domain_port);
         if ($port_state != GOOD_PORT) {
            printf ("[%d]$func_name:$$> Webserver $webserver_domain_name port $webserver_domain_port is bad\n", __LINE__);
            kill 9, $$;
         }
         
         # Attempt copying the webpage
         my $verbose = ($D & 0x8) ? "-v" : "";
         my $redirection = ($D & 0x8) ? "" : "2>/dev/null";
         $rc = system "$SCP_PGM -p $verbose -F $working_dir/host.conf $working_dir/webpage.html webpage:$webpage_remote_dir/$host.$yabs_dirlist.html $redirection";
         if ($rc) {
            printf ("[%d]$func_name:$$> ERROR: Webpage copy failed to $webserver_domain_name\n", __LINE__);
            kill 9, $$;
         }
         else {
#            printf ("[%d]$func_name:$$> Webpage copied to $webserver_domain_name  $webpage_remote_dir/$host.$yabs_dirlist.html\n", __LINE__);
            kill 9, $$;
         }
      }
      elsif ($! == EAGAIN) {
         # EAGAIN is the supposedly recoverable fork error
         sleep 5;
         redo WEB_PAGE_FORK;
      }
      else {
         # weird fork error occurred
         die "Can't fork: $!\n";
      }
   }
} # webpage


#------------------------------------------------------------------------------
# main: 
sub main 
{
   my $func_name="main";
   my $output;

   # Get and process command line arguments
   &process_cmd_line;

   # Need to select log destination and clear the logfile out
   &set_log_dest ("clear");

   # Display environment if debug is enabled
   if ( $D & 0x4 ) { &display_env }
   
   # Read options from the yabs.dirlist file
   &read_yabs_dirlist;

   # Display environment if debug is enabled
   if ( $D & 0x4 ) { &display_env }
   
   # Capture current conditions
   if ($test_mode == TRUE){
      print "--- TEST MODE ----------------- yabs $script_ver $script_date --------\n";
   }
   else {
      print "------------------------------- yabs $script_ver $script_date --------\n";
   }
   print "$host \"$backup_type\" backup log for file \"$yabs_dirlist\"\n";
   print "Email sent to: $email_address \n";
   print "Number of days backups kept: $keep_days (0: Keep all) \n";
   my $start_time = &get_time;
   print "Script started at: $start_time\n";
   print "----------------------------------------------------------------\n";
   print "Starting uptime, load average, and users:\n";
   my $w_cmd = `$W_PGM 2>&1`;
   print "$w_cmd";
   print "----------------------------------------------------------------\n";
   print "Starting disk usage:\n";
   # Use the yabs.df script file if it exists
   if (-e "yabs.df"){
      $output = `$working_dir/yabs.df 2>&1`;
      print "$output";
   }
   else {
      $output = `$DF_PGM 2>&1`;
      print "$output";
   }

   # Loop through and process all the entries in the $yabs_dirlist.  Error checking has
   # already been done in read_yabs_dirlist.
   my $pre_cmds;
   my $start_dir;
   my $post_cmds;
   open YABS_DIRLIST, "<", "$yabs_dirlist" or die "Cannot open $yabs_dirlist for reading: $!";
   while (<YABS_DIRLIST>) {
      if (/^PRE_CMDS:\s*(.*)/) {
         $pre_cmds = $1;
         while (<YABS_DIRLIST>) {
            if (/^START_DIR:\s*(.*)/) {
               $start_dir = $1;
               next;
            }

            if (/^DEST_DIR:\s*(.*)/) {
               $dest_dir = $1;
               next;
            }

            if (/^POST_CMDS:\s*(.*)/) {
               $post_cmds = $1;
               last;
            }
         }
         # All four lines have been read and stored.
         # Execute the pre backup commands
         my $output = `$pre_cmds 2>&1`;
         print "$output\n";

         # Start the backup.  
         $recurse_level = 0;
         if ("$backup_type" eq "rsync") {
            # rsync backups behave differently and use their own recursive function
            &purge_archives ($dest_dir, $backup_type, $keep_days, "none");
            &recurse_tree_rsync ($start_dir);

            # Compress the directory where rsync put the backups but only if --backup-dir was added as an option
            if ($compress_rsync_backup eq "yes") {
               if ($rsync_opts =~ m/.*--backup.*/) {
                  my $backup_dir = "";
                  my $filename = "";
                  my $command = "";
                  # Check if $dest_dir has a colon in it.
                  if ($dest_dir =~ m/(.*):(.*)/){
                     # The destination is on a remote host so use ssh to perform the command.
                     my $host_name = $1;
                     $backup_dir = dirname($2);
                     $filename = basename($2);
                     # Use ssh to perform the command.  Path to zip must be in the environment.
                     $command = "$SSH_PGM -F $working_dir/host.conf " . "$host_name " . "\'" . "cd \"$backup_dir\"; " . "zip -ryuq " . "\"${filename}.${backup_yymmddhhmm}.zip\" " . "\"${filename}.$backup_yymmddhhmm\"" . "\'";
                     my $output = `$command 2>&1`;
                     # Now that the zipfile has been made the individual files can be removed
                     $command = "$SSH_PGM -F $working_dir/host.conf " . "$host_name " . "\'" . "cd \"$backup_dir\"; " . "rm -fr " . "\"${filename}.${backup_yymmddhhmm}\" " . "\'";
                     $output = `$command 2>&1`;
                  }
                  else {
                     # The destination is on the local host
                     $backup_dir = dirname($dest_dir);
                     $filename = basename($dest_dir);
                     # Use the local zip command
                     chdir "$backup_dir";
                     $command = "$ZIP_PGM -ryuq " . "\"${filename}.${backup_yymmddhhmm}.zip\" " . "\"${filename}.$backup_yymmddhhmm\"";
                     my $output = `$command 2>&1`;
                     # Now that the zipfile has been made the individual files can be removed
                     $command = "rm -fr " . "\"${filename}.$backup_yymmddhhmm\"";
                     $output = `$command 2>&1`;
                  }
               }
            }
         }
         else {
            # Use this function for all the other backup types
            &recurse_tree ($start_dir);
         }

         # Execute the post backup commands
         $output = `$post_cmds 2>&1`;
         print "$output\n";
         next;
      }
   }
   close YABS_DIRLIST;

   # Return to the working directory in case currently in a subdirectory somewhere
   chdir "$working_dir";

   print "----------------------------------------------------------------\n";
   print "Ending uptime, load average, and users:\n";
   $w_cmd = `$W_PGM 2>&1`;
   print "$w_cmd";
   print "----------------------------------------------------------------\n";
   print "Ending disk usage:\n";
   # Use the yabs.df script file if it exists
   if (-e "yabs.df"){
      $output = `$working_dir/yabs.df 2>&1`;
      print "$output";
   }
   else {
      $output = `$DF_PGM 2>&1`;
      print "$output";
   }
   print "----------------------------------------------------------------\n";
   my $end_time = &get_time;
   print "Script ended at: $end_time\n";
   print "----------------------------------------------------------------\n";

   # Send the results via email if an email address has been supplied
   if ($email_address ne "none"){
      my $test_mode_status = "";
      if ($test_mode == TRUE){
         $test_mode_status = "Test Mode: ";
      }

      # Search the logfile for errors
      copy "$working_dir/logfile", "/tmp/logfile.$$";
      my $error_status = "";
      my $error_cnt = 0;
      my $line_number = 0;
      open SEARCH_ERR, "<", "/tmp/logfile.$$" or die "Cannot open /tmp/logfile.$$ for reading: $!";
      while (<SEARCH_ERR>){
         $line_number++;
         # Use the terms in @search_err that come from the yabs.dirlist file
         my $index = 0;
         while ($index < $search_err_index){
            if (/$search_err[$index++]/) {
               $error_cnt++;
               $error_status = "ERRORS($error_cnt) ";
               # Display the errors found unless turned off with a debug flag
               if (!($D & 0x200) ) { 
                  if ($error_cnt == 1){
                     # Only output this once and only if an error has been found
                     printf ("Below is a summary of errors found in the output above preceded by the line number:\n");
                  }
                  printf ("[$line_number]: $_");
               }
            }
         }
      }
      close SEARCH_ERR;
      unlink "/tmp/logfile.$$";

      # Send the email
      system "cat logfile | $MAIL_PGM -s \"yabs: $host $error_status$test_mode_status$backup_type backup for $yabs_dirlist\" $email_address";
   }

   # Update the webpage one last time if it is enabled
   &webpage;

} # main


#-------------------------------------------------------------------------------------------
# This is where everything actually starts happening
&set_working_dir;
&get_version;

# Check if -hd, -hs or -h on command line and display them 
$rc = GetOptions ( 
   'h|help' => sub {&display_help}, 
   'hd' => sub {&display_debug_help},
   'hs' => sub {&display_setup_help},
);

# Check if yabs is already running and bail out if it is
if (-e "$working_dir/$yabs_lock") {
   die "ERROR: yabs aborted. $working_dir/$yabs_lock file exists. Delete manually if it is old.\n";
}
else {
   open YABS_LOCK, ">", "$working_dir/$yabs_lock" or die "Cannot open $working_dir/$yabs_lock for writing: $!";
   close YABS_LOCK;
}

&setup_env;
&init_debug;
&main;

# Cleanup before exiting
close LOG;

# Delete the yabs.lock file so that yabs will run next time
unlink "$working_dir/$yabs_lock";
#printf ("debug[%d]$func_name:$$> \n", __LINE__);