#! /bin/csh -f
#
# Wrapper script for PGENESIS (parallel Genesis)
#
# USAGE:
# pgenesis [-config <spec>] # where spec is a filename containing the
# # list of machines to be used (1 per line)
# # this should be in the format of a PVM
# # hostfile, since it will be passed to the
# # PVM daemon upon startup
# [-skip] # says skip all the checking and cleaning
# # just execute parallel genesis
# [-nohalt] # says don't halt pvm or clean up after
# # pgenesis has run
# [-v] # verbose -- turns on lots of reporting
# [-nox] # says run without Xodus
# [-help] # prints usage information for pgenesis
# [-debug [tty] # specifies what style of debugging to use
# [dbx] # when running workers; tty = each worker
# [gdb]] # gets its own tty window, dbx = use dbx
# [totalview] # debugger on workers, gdb = use GNU gdb
# # debugger on workers, totalview = use
# # Cray totalview debugger
# [-nodes <num>] # (PARAGON and T3D/E-only) specifies
# # partition size to use
# [-partition <name> # (PARAGON only) partition to use
# # (default: open)
# [-browse] # attempt to run netscape on the html docs
#
# Users may also modify the behavior of this script by setting the following
# environment variables:
# PGENESIS_ROOT: specifies the directory
# PVM_ROOT: specifies which PVM to use
#
# Copyright (c) 1997 Pittsburgh Supercomputing Center
#
# Written by Nigel Goddard and Greg Hood, PSC
#
# HISTORY
# 1996 -- written by Nigel Goddard
# Apr 97 -- made more site-independent, Greg Hood
# Jun 97 -- Paragon support, Nigel Goddard
# Jul 97 -- user extended PGENESIS support, Nigel Goddard
# Aug 97 -- improved default handling, Greg Hood
# May 00 -- added "quit" to all piped commands to pvm console to
# avoid segmentation faults on LINUX, Greg Hood
# Dec 01 -- removed "quit" because it no longer seems to be needed
# with LINUX, and actually breaks things on other
# architectures; also added "sleep 1" after pvm daemon
# startup because there is apparently a race condition
# where pgenesis will generate "pvm_trecv(): Can't contact
# local daemon" messages if it starts up before the
# pvm daemon has finished its own initialization, Greg Hood
# Aug 05 -- removed support for Paragon, Cray T3D/T3E; added support
# for MPI-based architectures, including Cray XT3
#
# Phases:
# 1. determine default directories
# 2. determine architecture type
# 3. set general defaults
# 4. set architecture-specific defaults
# 5. process command line arguments
# 6. general initialization
# 7. architecture-specific initialization
# 8. check that .cshrc or .bashrc doesn't print to tty
# 9. check pvm links are set and accessible
# 10. make sure .simrc file is present
# 11. check .rhosts set correctly
# 12. start pvm daemon on requested cpus
# 13. run pgenesis with supplied args
# 14. shut down pvm daemon
#---------------------------------------------------------------------------*
# VARIABLES WHICH MAY BE MODIFIED AFTER INSTALLATION AT PARTICULAR SITES
# the installation process sets these default locations to actual
# directories when creating the pgenesis script, so it is likely that
# nothing will have to be modified
#---------------------------------------------------------------------------*
# PGENESIS_SRC: the location of vanilla PGENESIS source
setenv PGENESIS_SRC /data2/software/genesis-2.4-openmpi-1.8.8/pgenesis
# PGENESIS_INST: the location of vanilla PGENESIS installation (libs, bins)
setenv PGENESIS_INST /data2/software/genesis-2.4-openmpi-1.8.8/pgenesis
# PGENESIS_EXTSRC: the location of extended PGENESIS source
setenv PGENESIS_EXTSRC _extpgenesis-src_
# PGENESIS_EXTLIBS: the names of the extension library directories
setenv PGENESIS_EXTLIBS _extpgenesis-libs_
# PGENESIS_EXTINST: location of extended PGENESIS installation (libs, bins)
setenv PGENESIS_EXTINST _extpgenesis-inst_
# GENESIS_DEFAULT: the location for serial genesis
setenv GENESIS_DEFAULT /data2/software/genesis-2.4-openmpi-1.8.8/genesis
# USE_MPI: set to 1 if we are using MPI
setenv USE_MPI 01
# USE_PVM: set to 1 if we are using PVM
setenv USE_PVM 0
# PVM_DEFAULT: the PVM root directory on the local machine
setenv PVM_DEFAULT /data2/software/genesis-2.4-openmpi-1.8.8/pgenesis
# PVM_ARCH: the PVM architecture type
setenv PVM_ARCH Linux
# PGENESIS_TMP: the area pgenesis uses for scratch files
setenv PGENESIS_TMP /tmp
#---------------------------------------------------------------------------*
# Phase 1: determine default directories
# PGENESIS_ROOT
# PGENESIS_CODE
# GENESIS_ROOT
# PVM_ROOT
#---------------------------------------------------------------------------*
# if PGENESIS_ROOT is not defined, assume it is one directory above where
# this executable was found
if (! $?PGENESIS_ROOT) then
set executable=$0
if ($executable == pgenesis) then
set thisdir=.
else
set thisdir=$executable:h
endif
setenv PGENESIS_ROOT `cd $thisdir/..; pwd`
endif
# if these variables were not set by installation, then set them here
if ( `echo $PGENESIS_SRC | sed s/_/+/g` == +pgenesis-src+ ) then
setenv PGENESIS_SRC $PGENESIS_ROOT
endif
if ( `echo $PGENESIS_INST | sed s/_/+/g` == +pgenesis-inst+ ) then
setenv PGENESIS_INST $PGENESIS_ROOT
endif
if ( `echo $PVM_DEFAULT | sed s/_/+/g` == +pvm+ ) then
setenv PVM_DEFAULT $PGENESIS_ROOT/../pvm3
endif
# PGENESIS_ROOT is where the binaries are held
# PGENESIS_CODE is a path list for gdb to find sources
if ( `echo $PGENESIS_EXTINST | sed s/_/+/g` != +extpgenesis-inst+ ) then
setenv PGENESIS_ROOT $PGENESIS_EXTINST
setenv PGENESIS_CODE $PGENESIS_SRC/src/par
foreach dir ($PGENESIS_EXTLIBS)
setenv PGENESIS_CODE "$PGENESIS_EXTSRC/$dir"\:"$PGENESIS_CODE"
end
else
setenv PGENESIS_ROOT $PGENESIS_INST
setenv PGENESIS_CODE $PGENESIS_SRC/src/par
endif
if (! -r $PGENESIS_ROOT) then
echo "PGENESIS_ROOT (currently $PGENESIS_ROOT) must be set to a"
echo "valid directory"
exit
endif
if (! -r $PGENESIS_TMP) then
echo "PGENESIS_TMP (currently $PGENESIS_TMP) must be set to a"
echo "valid directory"
exit
endif
if (! $?GENESIS_ROOT) then
# try to set it
if ( `echo $GENESIS_DEFAULT | sed s/_/+/g` == +genesis+ ) then
if ( -r $PGENESIS_ROOT/../genesis ) then
setenv GENESIS_ROOT $PGENESIS_ROOT/../genesis
else
echo "Warning: GENESIS_ROOT not set. You need to set
it to the location"
echo "of serial GENESIS. Otherwise serial GENESIS scripts will be unavailable."
setenv GENESIS_ROOT $PGENESIS_ROOT
endif
else
setenv GENESIS_ROOT $GENESIS_DEFAULT
endif
endif
# if the user has not explicitly specified PVM_ROOT
# then the defaults will be used
if ( $USE_PVM ) then
if (! $?PVM_ROOT) then
setenv PVM_ROOT $PVM_DEFAULT
endif
if (! -r $PVM_ROOT) then
echo "PVM_ROOT (currently $PVM_ROOT) must be set to a"
echo "valid directory"
exit
endif
endif
#---------------------------------------------------------------------------*
# Phase 2: determine architecture type
#---------------------------------------------------------------------------*
if ( $USE_PVM) then
if (! $?PVM_ARCH) then
if (-x $PVM_ROOT/lib/pvmgetarch) then
setenv PVM_ARCH `$PVM_ROOT/lib/pvmgetarch`
else
echo "Cannot execute $PVM_ROOT/lib/pvmgetarch"
exit
endif
endif
test $PVM_ARCH != ALPHA
set alpha=$status
test $PVM_ARCH != SGI5
set sgi5=$status
endif
#---------------------------------------------------------------------------*
# Phase 3: set general defaults
#---------------------------------------------------------------------------*
set debug=
set vrbose=0
set nox=0
set single=1
set nargv=
set nargc=0
set simrc=.psimrc
set altsimrc=0
set config=
set skip=0
set nohalt=0
set tmp=$PGENESIS_TMP/pgenesis.1.$$
set tmp2=$PGENESIS_TMP/pgenesis.2.$$
set remote=
#---------------------------------------------------------------------------*
# Phase 4: set architecture-specific defaults
#---------------------------------------------------------------------------*
#---------------------------------------------------------------------------*
# Phase 5: process command line arguments
#---------------------------------------------------------------------------*
while ($#argv > 0)
switch ($1)
case -help:
echo "pgenesis takes the usual genesis flags plus these additional flags:"
if ($USE_MPI) then
echo " -nodes <num> = specifies number of nodes to use"
else
echo " -config <file> = specifies a file listing the cpus to use"
endif
echo " -nohalt = don't halt PVM after pgenesis runs (-skip can be used next time"
echo " -skip = skip checks, just run pgenesis and exit"
echo " -v = show important parts of execution of this script"
echo " -nox = run without Xodus"
echo " -debug [tty|dbx|gdb|totalview] = each node has a window:"
echo " plain, under dbx, under emacs/gdb, or via totalview"
echo " -help = print this information"
echo " -browse = browse the PGENESIS documentation with netscape"
exit
breaksw
case -debug:
shift
if ($1 == dbx || $1 == tty || $1 == gdb || $1 == totalview) then
set debug=$1
else
echo "valid debug modes are tty, dbx, gdb and totalview, $1 not recognized, using tty"
set debug="tty"
endif
setenv PGENESIS_DEBUG 1
breaksw
case -nox:
set nox=1
breaksw
case -skip:
set skip=1
breaksw
case -nohalt:
set nohalt=1
breaksw
case -config:
shift
set single=0
# strip comments and change * to @
set p=`cat $1 | sed 's/#.*//g' | sed s/'*'/@/g`
# strip out components with * or @ or =, leaving cpus
while ($#p > 0)
echo $p[1] | grep '[@=]' >& /dev/null
if ($status) then
set remote = "$remote $p[1]"
endif
shift p
end
set config=$1
breaksw
case -nodes:
if ($USE_MPI) then
shift
set num_nodes=$1
else
echo -nodes option is only valid for MPI-based installations
endif
breaksw
case -altsimrc:
set altsimrc=1
set nargv="$nargv $1 $2"
@ nargc+=2
shift
set simrc=$1
breaksw
case -v:
set vrbose=1
breaksw
case -browse:
netscape $PGENESIS_ROOT/Hyperdoc/welcome.html &
exit
breaksw
default:
set nargv="$nargv $1"
@ nargc++
breaksw
endsw
shift
end
#---------------------------------------------------------------------------*
# Phase 6: general initialization
#---------------------------------------------------------------------------*
if ($USE_PVM) then
if ($vrbose) then
echo PVM_ROOT: $PVM_ROOT
echo PVM_ARCH: $PVM_ARCH
endif
endif
# add default .psimrc if not specified
if (! $altsimrc) then
set nargv="-altsimrc .psimrc $nargv"
@ nargc+=2
endif
#check that GENESIS_ROOT is valid if debugging with gdb
if (x$debug == xgdb && ! -r $GENESIS_ROOT) then
echo "GENESIS_ROOT (currently $GENESIS_ROOT) must be set to a"
echo "valid directory"
exit
endif
#---------------------------------------------------------------------------*
# Phase 7: architecture-specific initialization
#---------------------------------------------------------------------------*
#---------------------------------------------------------------------------*
# Phase 8: check that .cshrc doesn't produce output
#---------------------------------------------------------------------------*
if (! $skip) then
if ($vrbose) then
echo number of args: $nargc
echo which are: $nargv
echo simrc is: $simrc
echo remote cpus: $remote
endif
echo "performing checks..."
if ($USE_PVM) then
if ($vrbose) then
echo "checking .cshrc ..."
endif
csh -f $HOME/.cshrc </dev/null >$tmp
if ( ! -z $tmp) then
echo 'Your ~/.cshrc produces output for non-interactive shells\!'
echo 'Edit your .cshrc to remedy this, e.g. add'
echo ' if (! $?prompt ) then'
echo ' ...'
echo ' endif'
echo 'around the output statements'
echo 'To check, verify that "(csh -f ~/.cshrc)" prints nothing'
rm -f $tmp
exit
rm -f $tmp
endif
endif
#---------------------------------------------------------------------------*
# Phase 9: check pvm links are set and have proper permissions *
#---------------------------------------------------------------------------*
if (! $skip) then
if ($USE_PVM) then
if (! $?PVM_EXPORT) then
setenv PVM_EXPORT DISPLAY
endif
setenv PVM $PVM_ROOT/lib/pvm
if (! -x $PVM) then
echo "can't find pvm executable: $PVM"
exit
endif
# set the debugger if it isn't set
if (! $?PVM_DEBUGGER) then
if (x$debug != x) then # set it to a PGENESIS debugger
setenv PVM_DEBUGGER $PGENESIS_ROOT/lib/$debug"debugger"
else # set it to the default debugger
setenv PVM_DEBUGGER debugger
endif
endif
if (! -x $PVM_DEBUGGER && ! -x $PVM_ROOT/lib/$PVM_DEBUGGER) then
echo "can't find executable pvm debugger : $PVM_DEBUGGER"
echo " nor: $PVM_ROOT/lib/$PVM_DEBUGGER"
exit
endif
setenv PVM_EXPORT "PVM_DEBUGGER:$PVM_EXPORT"
if ($vrbose) then
echo "using $PVM_DEBUGGER to debug workers..."
echo "PVM_EXPORT is: $PVM_EXPORT"
endif
if ($vrbose) then
echo "checking executables are accessible to PVM..."
endif
# make PVM_ROOT directories as needed
if ( ! -d ~/pvm3) then
mkdir -p ~/pvm3
endif
setenv MY_PVM_ROOT ~/pvm3
# link executables to where PVM can find them
if ( -d $PGENESIS_ROOT/bin/$PVM_ARCH) then
if (! -d $MY_PVM_ROOT/bin/$PVM_ARCH) then
mkdir -p $MY_PVM_ROOT/bin/$PVM_ARCH
endif
foreach exe (minpgenesis nxpgenesis pgenesis)
if (-x $PGENESIS_ROOT/bin/$PVM_ARCH/$exe ) then
if ($vrbose) then
echo "ln -s $PGENESIS_ROOT/bin/$PVM_ARCH/$exe $MY_PVM_ROOT/bin/$PVM_ARCH/$exe"
endif
rm -f $MY_PVM_ROOT/bin/$PVM_ARCH/$exe
ln -s $PGENESIS_ROOT/bin/$PVM_ARCH/$exe $MY_PVM_ROOT/bin/$PVM_ARCH/$exe
endif
end
endif
# set AFS permissions if this appears to be an AFS directory
set lead=`(cd $MY_PVM_ROOT; pwd) | sed s+/+' '+g | awk '{print $1}'`
if ($lead == "afs") then
if ($vrbose) then
echo "checking afs permissions"
endif
if (-x /usr/afsws/bin/fs) then
set fs=/usr/afsws/bin/fs
else
set fs=fs
endif
$fs la $HOME >& $tmp
if ($status) then
echo "unable to read acls on $HOME"
exit
endif
cat $tmp | grep system:anyuser | grep rl >& /dev/null
if ($status) then
echo "Your home directory may need to be made world-readable for pvm"
endif
foreach d (~/pvm3 ~/pvm3/bin ~/pvm3/bin/$PVM_ARCH)
$fs sa $d "system:anyuser" rl
end
endif
endif
endif
#---------------------------------------------------------------------------*
# Phase 10: make sure .simrc file is present *
#---------------------------------------------------------------------------*
if (! $skip) then
if ($vrbose) then
echo checking for simrc ...
endif
if (! -r $simrc && ! -r $HOME/$simrc ) then
if ($altsimrc) then
echo "can't find specified .simrc: $simrc"
exit
else
echo "copying default .psimrc to ./.psimrc"
cp $PGENESIS_INST/startup/.psimrc ./.psimrc
endif
else
if ($vrbose) then
echo "found simrc"
endif
endif
endif
#---------------------------------------------------------------------------*
# Phase 11: check .rhosts set appropriately
#---------------------------------------------------------------------------*
# obsolete -- everyone should be using ssh-based PVM for security reasons!
#---------------------------------------------------------------------------*
# Phase 12: start pvm daemon
#---------------------------------------------------------------------------*
if (! $skip && $USE_PVM) then
if ($vrbose) then
echo "starting PVM daemon ($PVM)..."
endif
# catch interrupts, shut down pvmd and therefore all workers
onintr shutdown
# start the local pvm daemon, obtain configuration
echo "conf" | $PVM $config >& $tmp
if ($vrbose) then
echo "version" | $PVM
endif
grep ' running' $tmp >& /dev/null
if ($status) then
echo "$PVM failed to start pvm daemon on" `hostname`
rm -f $tmp
exit
endif
# if multiple cpus, clean up ones not already running pvm daemon
if (! $single) then
if ($vrbose) then
echo "checking if pvm already running on remote hosts..."
endif
foreach m ($remote)
grep $m $tmp >& /dev/null
if ($status) then
if ($vrbose) then
echo "cleaning on $m..."
endif
ssh $m "rm -f /tmp/pvml.* /tmp/pvmd.*" >& /dev/null
endif
end
endif
# now try to start the daemon on all remote cpus
if (! $single) then
echo "add $remote" | $PVM >& $tmp
foreach m ($remote)
grep $m $tmp >& /dev/null
if ($status) then
echo "pvm daemon failed to start on $m, continuing..."
endif
end
endif
if ($vrbose) then
echo "conf" | $PVM >& $tmp
echo "PVM console reports daemon running on these hosts:"
cat $tmp
endif
if ($vrbose) then
echo resetting PVM daemons...
endif
echo "reset" | $PVM >& /dev/null
rm -f $tmp
sleep 1
endif
#---------------------------------------------------------------------------*
# Phase 13: fire up genesis
#---------------------------------------------------------------------------*
if (! $skip) then
echo "starting pgenesis executable"
endif
if ($nox) then
set exec=$PGENESIS_ROOT/bin/$PVM_ARCH/nxpgenesis
else
set exec=$PGENESIS_ROOT/bin/$PVM_ARCH/pgenesis
endif
if ($USE_MPI) then
if ( x$debug == x) then
echo Going to run on $num_nodes nodes: $exec $nargv
# MPI_CMD
mpirun --verbose --mca plm_rsh_agent "/usr/bin/ssh" --host rauchbier -np $num_nodes nice -n10 $exec $nargv
else
echo Going to debug on $num_nodes nodes: $exec $nargv
# MPI_DEBUG_CMD
mpirun -gdb -np $num_nodes $exec $nargv
endif
else if ($USE_PVM) then
# run it under the debugger or not
if ( x$debug == x || $debug == tty) then
if ($vrbose) then
echo $exec $nargv
endif
$exec $nargv
else if ( $debug == gdb) then
echo args are $nargv
set TEMPCMD=$PGENESIS_TMP/debugger.$$
set INITCMD=$PGENESIS_TMP/debuginit.$$
echo dir "$PGENESIS_CODE"\:"$GENESIS_ROOT/src/sim"\:"$GENESIS_ROOT/src/sys"\:"$GENESIS_ROOT/src/ss"\:"$GENESIS_ROOT/src/shell"\:"$GENESIS_ROOT/src/newconn" >> $INITCMD
echo set args $nargv >> $INITCMD
echo show args >> $INITCMD
echo cd `pwd` >> $INITCMD
echo "(defun rungdb ()" > $TEMPCMD
echo " (gdb "'"'gdb --command=$INITCMD $exec'"'"))" >> $TEMPCMD
emacs -fn 9x15 -geometry 80x40 -T PGENESIS-MASTER@`hostname` -l $TEMPCMD -f rungdb >& /dev/null
rm -f $TEMPCMD $INITCMD
else
echo "use these flags: $nargv"
$debug $exec
endif
else
echo "Installation error: neither USE_MPI nor USE_PVM is set"
endif
#---------------------------------------------------------------------------*
# Phase 14: clean up
#---------------------------------------------------------------------------*
shutdown:
if ($USE_PVM && ! $skip && ! $nohalt) then
echo "halting PVM daemon and cleaning up ..."
echo halt | $PVM >& /dev/null
rm -f /tmp/pvml.* /tmp/pvmd.* >& /dev/null
if (! $single) then
foreach m ($remote)
ssh $m "rm -f /tmp/pvml.* /tmp/pvmd.*" >& /dev/null
end
endif
if ($PVM_ARCH == ALPHAMP || $PVM_ARCH == HPPAMP || \
$PVM_ARCH == SUNMP || $PVM_ARCH == SGIMP64) then
$PGENESIS_INST/bin/ipclean
endif
endif
|