#! /bin/csh -f
#
# Wrapper script for PGENESIS (parallel Genesis)
#
# USAGE:
#   pgenesis [-config <spec>]     # where spec is a filename containing the
#                                 #   list of machines to be used (1 per line)
#				  #   this should be in the format of a PVM
#				  #   hostfile, since it will be passed to the
#				  #   PVM daemon upon startup
#            [-skip]              # says skip all the checking and cleaning
#				  #   just execute parallel genesis
#            [-nohalt]            # says don't halt pvm or clean up after
#                                 #   pgenesis has run
#            [-v]                 # verbose -- turns on lots of reporting
#            [-nox]               # says run without Xodus
#            [-help]              # prints usage information for pgenesis
#            [-debug [tty]	  # specifies what style of debugging to use
#                    [dbx]        #    when running workers; tty = each worker
#                    [gdb]]       #    gets its own tty window, dbx = use dbx
#		     [totalview]  #    debugger on workers, gdb = use GNU gdb
#                                 #    debugger on workers, totalview = use
#				  #    Cray totalview debugger
#	     [-nodes <num>]       # (PARAGON and T3D/E-only) specifies
#				  #    partition size to use
#	     [-partition <name>   # (PARAGON only) partition to use
#				  #    (default: open)
#	     [-browse]		  # attempt to run netscape on the html docs
#
#  Users may also modify the behavior of this script by setting the following
#         environment variables:
#     PGENESIS_ROOT:  specifies the directory
#     PVM_ROOT:	      specifies which PVM to use

# 
# Copyright (c) 1997 Pittsburgh Supercomputing Center
#
# Written by Nigel Goddard and Greg Hood, PSC
# 
# HISTORY
#	1996   -- written by Nigel Goddard
#	Apr 97 -- made more site-independent, Greg Hood
#	Jun 97 -- Paragon support, Nigel Goddard
#	Jul 97 -- user extended PGENESIS support, Nigel Goddard
#       Aug 97 -- improved default handling, Greg Hood
#       May 00 -- added "quit" to all piped commands to pvm console to
#                   avoid segmentation faults on LINUX, Greg Hood
#       Dec 01 -- removed "quit" because it no longer seems to be needed
#                   with LINUX, and actually breaks things on other
#                   architectures; also added "sleep 1" after pvm daemon
#                   startup because there is apparently a race condition
#                   where pgenesis will generate "pvm_trecv(): Can't contact
#                   local daemon" messages if it starts up before the
#                   pvm daemon has finished its own initialization, Greg Hood
#       Aug 05 -- removed support for Paragon, Cray T3D/T3E; added support
#                   for MPI-based architectures, including Cray XT3
#
# Phases:
# 1. determine default directories
# 2. determine architecture type
# 3. set general defaults
# 4. set architecture-specific defaults
# 5. process command line arguments
# 6. general initialization
# 7. architecture-specific initialization
# 8. check that .cshrc or .bashrc doesn't print to tty
# 9. check pvm links are set and accessible
# 10. make sure .simrc file is present
# 11. check .rhosts set correctly
# 12. start pvm daemon on requested cpus
# 13. run pgenesis with supplied args
# 14. shut down pvm daemon


#---------------------------------------------------------------------------*
# VARIABLES WHICH MAY BE MODIFIED AFTER INSTALLATION AT PARTICULAR SITES
#      the installation process sets these default locations to actual
#      directories when creating the pgenesis script, so it is likely that
#      nothing will have to be modified
#---------------------------------------------------------------------------*

#     PGENESIS_SRC: the location of vanilla PGENESIS source
setenv PGENESIS_SRC /data2/software/genesis-2.4-openmpi-1.8.8/pgenesis

#     PGENESIS_INST: the location of vanilla PGENESIS installation (libs, bins)
setenv PGENESIS_INST /data2/software/genesis-2.4-openmpi-1.8.8/pgenesis

#     PGENESIS_EXTSRC: the location of extended PGENESIS source
setenv PGENESIS_EXTSRC _extpgenesis-src_

#     PGENESIS_EXTLIBS: the names of the extension library directories
setenv PGENESIS_EXTLIBS _extpgenesis-libs_

#     PGENESIS_EXTINST: location of extended PGENESIS installation (libs, bins)
setenv PGENESIS_EXTINST _extpgenesis-inst_

#     GENESIS_DEFAULT: the location for serial genesis
setenv GENESIS_DEFAULT	/data2/software/genesis-2.4-openmpi-1.8.8/genesis

#     USE_MPI: set to 1 if we are using MPI
setenv USE_MPI          01

#     USE_PVM: set to 1 if we are using PVM
setenv USE_PVM          0

#     PVM_DEFAULT: the PVM root directory on the local machine
setenv PVM_DEFAULT	/data2/software/genesis-2.4-openmpi-1.8.8/pgenesis

#     PVM_ARCH: the PVM architecture type
setenv PVM_ARCH		Linux

#     PGENESIS_TMP: the area pgenesis uses for scratch files
setenv PGENESIS_TMP	/tmp


#---------------------------------------------------------------------------*
# Phase 1: determine default directories
#            PGENESIS_ROOT
#            PGENESIS_CODE
#            GENESIS_ROOT
#            PVM_ROOT
#---------------------------------------------------------------------------*

# if PGENESIS_ROOT is not defined, assume it is one directory above where
# this executable was found
if (! $?PGENESIS_ROOT) then
    set executable=$0
    if ($executable == pgenesis) then
	set thisdir=.
    else
	set thisdir=$executable:h
    endif
    setenv PGENESIS_ROOT `cd $thisdir/..; pwd`
endif

#   if these variables were not set by installation, then set them here
if ( `echo $PGENESIS_SRC | sed s/_/+/g` == +pgenesis-src+ ) then
    setenv PGENESIS_SRC $PGENESIS_ROOT
endif
if ( `echo $PGENESIS_INST | sed s/_/+/g` == +pgenesis-inst+ ) then
    setenv PGENESIS_INST $PGENESIS_ROOT
endif
if ( `echo $PVM_DEFAULT | sed s/_/+/g` == +pvm+ ) then
    setenv PVM_DEFAULT $PGENESIS_ROOT/../pvm3
endif

#	PGENESIS_ROOT is where the binaries are held
#	PGENESIS_CODE is a path list for gdb to find sources
if ( `echo $PGENESIS_EXTINST | sed s/_/+/g` != +extpgenesis-inst+ ) then
	setenv PGENESIS_ROOT $PGENESIS_EXTINST
	setenv PGENESIS_CODE $PGENESIS_SRC/src/par
	foreach dir ($PGENESIS_EXTLIBS)
		setenv PGENESIS_CODE "$PGENESIS_EXTSRC/$dir"\:"$PGENESIS_CODE"
	end
else
	setenv PGENESIS_ROOT $PGENESIS_INST	
	setenv PGENESIS_CODE $PGENESIS_SRC/src/par
endif

if (! -r $PGENESIS_ROOT) then
	echo "PGENESIS_ROOT (currently $PGENESIS_ROOT) must be set to a"
	echo "valid directory"
	exit
endif

if (! -r $PGENESIS_TMP) then
	echo "PGENESIS_TMP (currently $PGENESIS_TMP) must be set to a"
	echo "valid directory"
	exit
endif

if (! $?GENESIS_ROOT) then
# try to set it
	if ( `echo $GENESIS_DEFAULT | sed s/_/+/g` == +genesis+ ) then
		if ( -r $PGENESIS_ROOT/../genesis ) then
			setenv GENESIS_ROOT $PGENESIS_ROOT/../genesis
		else
			echo "Warning: GENESIS_ROOT not set.  You need to set
it to the location"
			echo "of serial GENESIS.  Otherwise serial GENESIS scripts will be unavailable."
			setenv GENESIS_ROOT $PGENESIS_ROOT
		endif
	else
		setenv GENESIS_ROOT $GENESIS_DEFAULT
	endif
endif

#     if the user has not explicitly specified PVM_ROOT
#	then the defaults will be used
if ( $USE_PVM ) then
  if (! $?PVM_ROOT) then
	setenv PVM_ROOT $PVM_DEFAULT
  endif
  if (! -r $PVM_ROOT) then
	echo "PVM_ROOT (currently $PVM_ROOT) must be set to a"
	echo "valid directory"
	exit
  endif
endif

#---------------------------------------------------------------------------*
# Phase 2: determine architecture type
#---------------------------------------------------------------------------*

if ( $USE_PVM) then
  if (! $?PVM_ARCH) then
    if (-x $PVM_ROOT/lib/pvmgetarch) then
        setenv PVM_ARCH `$PVM_ROOT/lib/pvmgetarch`
    else
        echo "Cannot execute $PVM_ROOT/lib/pvmgetarch"
        exit
    endif
  endif

  test $PVM_ARCH != ALPHA
  set alpha=$status
  test $PVM_ARCH != SGI5
  set sgi5=$status
endif


#---------------------------------------------------------------------------*
# Phase 3: set general defaults
#---------------------------------------------------------------------------*

set debug=
set vrbose=0
set nox=0
set single=1
set nargv=
set nargc=0
set simrc=.psimrc
set altsimrc=0
set config=
set skip=0
set nohalt=0
set tmp=$PGENESIS_TMP/pgenesis.1.$$
set tmp2=$PGENESIS_TMP/pgenesis.2.$$
set remote=

#---------------------------------------------------------------------------*
# Phase 4: set architecture-specific defaults
#---------------------------------------------------------------------------*

#---------------------------------------------------------------------------*
# Phase 5: process command line arguments
#---------------------------------------------------------------------------*

while ($#argv > 0)
	switch ($1)
	case -help:
		echo "pgenesis takes the usual genesis flags plus these additional flags:"
		if ($USE_MPI) then
		  echo "  -nodes <num>  = specifies number of nodes to use"
		else
		  echo "  -config <file> =  specifies a file listing the cpus to use"
		endif
		echo "  -nohalt  = don't halt PVM after pgenesis runs (-skip can be used next time"
		echo "  -skip  = skip checks, just run pgenesis and exit"
		echo "  -v  = show important parts of execution of this script"
		echo "  -nox  = run without Xodus"
		echo "  -debug [tty|dbx|gdb|totalview] = each node has a window:"
		echo "         plain, under dbx, under emacs/gdb, or via totalview"
		echo "  -help  = print this information"
		echo "  -browse  = browse the PGENESIS documentation with netscape"
		exit
		breaksw
	case -debug:
		shift
		if ($1 == dbx || $1 == tty || $1 == gdb || $1 == totalview) then
			set debug=$1
		else
			echo "valid debug modes are tty, dbx, gdb and totalview, $1 not recognized, using tty"
			set debug="tty"
		endif
		setenv PGENESIS_DEBUG 1
		breaksw
	case -nox:
		set nox=1
		breaksw
	case -skip:
		set skip=1
		breaksw
	case -nohalt:
		set nohalt=1
		breaksw
	case -config:
		shift
		set single=0
		# strip comments and change * to @
		set p=`cat $1 | sed 's/#.*//g' | sed s/'*'/@/g`
		# strip out components with * or @ or =, leaving cpus
		while ($#p > 0)
		  echo $p[1] | grep '[@=]' >& /dev/null
		  if ($status) then
		    set remote = "$remote $p[1]"
                  endif
		  shift p
		end
		set config=$1
		breaksw
	case -nodes:
	        if ($USE_MPI) then
		  shift
		  set num_nodes=$1
  		else
                  echo -nodes option is only valid for MPI-based installations
		endif
		breaksw
	case -altsimrc:
		set altsimrc=1
		set nargv="$nargv $1 $2"
		@ nargc+=2
		shift
		set simrc=$1
		breaksw
	case -v:
		set vrbose=1
		breaksw
	case -browse:
		netscape $PGENESIS_ROOT/Hyperdoc/welcome.html &
		exit
		breaksw
	default:
		set nargv="$nargv $1"
		@ nargc++
		breaksw
	endsw
	shift
end


#---------------------------------------------------------------------------*
# Phase 6: general initialization
#---------------------------------------------------------------------------*

if ($USE_PVM) then
  if ($vrbose) then
	echo PVM_ROOT: $PVM_ROOT
	echo PVM_ARCH: $PVM_ARCH
  endif
endif

# add default .psimrc if not specified
if (! $altsimrc) then
	set nargv="-altsimrc .psimrc $nargv"
	@ nargc+=2
endif

#check that GENESIS_ROOT is valid if debugging with gdb
if (x$debug == xgdb && ! -r $GENESIS_ROOT) then
    echo "GENESIS_ROOT (currently $GENESIS_ROOT) must be set to a"
    echo "valid directory"
    exit
endif

#---------------------------------------------------------------------------*
# Phase 7: architecture-specific initialization
#---------------------------------------------------------------------------*

#---------------------------------------------------------------------------*
# Phase 8: check that .cshrc doesn't produce output
#---------------------------------------------------------------------------*
if (! $skip) then
  if ($vrbose) then
	echo number of args: $nargc
	echo which are: $nargv
	echo simrc is: $simrc
	echo remote cpus: $remote
  endif

  echo "performing checks..."	
  if ($USE_PVM) then
    if ($vrbose) then
	echo "checking .cshrc ..."
    endif
    csh -f $HOME/.cshrc  </dev/null >$tmp
    if ( ! -z $tmp) then
        echo 'Your ~/.cshrc produces output for non-interactive shells\!'
  	echo 'Edit your .cshrc to remedy this, e.g. add'
	echo '     if (! $?prompt ) then'
	echo '         ...'
        echo '     endif'
        echo 'around the output statements'
	echo 'To check, verify that "(csh -f ~/.cshrc)" prints nothing'
	rm -f $tmp
	exit
    rm -f $tmp
  endif
endif

#---------------------------------------------------------------------------*
# Phase 9: check pvm links are set and have proper permissions              *
#---------------------------------------------------------------------------*
if (! $skip) then
  if ($USE_PVM) then
    if (! $?PVM_EXPORT) then
	setenv PVM_EXPORT DISPLAY
    endif
    setenv PVM $PVM_ROOT/lib/pvm
    if (! -x $PVM) then
  	echo "can't find pvm executable: $PVM"
	exit
    endif
    # set the debugger if it isn't set
    if (! $?PVM_DEBUGGER) then
	if (x$debug != x) then	# set it to a PGENESIS debugger
		setenv PVM_DEBUGGER $PGENESIS_ROOT/lib/$debug"debugger"
	else			# set it to the default debugger
		setenv PVM_DEBUGGER debugger
	endif
    endif
    if (! -x $PVM_DEBUGGER && ! -x $PVM_ROOT/lib/$PVM_DEBUGGER) then
	echo "can't find executable pvm debugger : $PVM_DEBUGGER"
	echo "	nor: $PVM_ROOT/lib/$PVM_DEBUGGER"
	exit
    endif

    setenv PVM_EXPORT "PVM_DEBUGGER:$PVM_EXPORT"
    if ($vrbose) then
	echo "using $PVM_DEBUGGER to debug workers..."
	echo "PVM_EXPORT is: $PVM_EXPORT"
    endif

    if ($vrbose) then
  	echo "checking executables are accessible to PVM..."
    endif

    # make PVM_ROOT directories as needed
    if ( ! -d ~/pvm3) then
  	mkdir -p ~/pvm3
    endif
    setenv MY_PVM_ROOT ~/pvm3

    # link executables to where PVM can find them
    if ( -d $PGENESIS_ROOT/bin/$PVM_ARCH) then
      if (! -d $MY_PVM_ROOT/bin/$PVM_ARCH) then
		mkdir -p $MY_PVM_ROOT/bin/$PVM_ARCH
      endif
      foreach exe (minpgenesis nxpgenesis pgenesis)
	if (-x $PGENESIS_ROOT/bin/$PVM_ARCH/$exe ) then
	  if ($vrbose) then
	    echo "ln -s $PGENESIS_ROOT/bin/$PVM_ARCH/$exe $MY_PVM_ROOT/bin/$PVM_ARCH/$exe"
          endif
          rm -f $MY_PVM_ROOT/bin/$PVM_ARCH/$exe
	  ln -s $PGENESIS_ROOT/bin/$PVM_ARCH/$exe $MY_PVM_ROOT/bin/$PVM_ARCH/$exe
	endif
      end
    endif

    # set AFS permissions if this appears to be an AFS directory
    set lead=`(cd $MY_PVM_ROOT; pwd) | sed s+/+' '+g | awk '{print $1}'`
    if ($lead == "afs") then
	if ($vrbose) then
		echo "checking afs permissions"
	endif
	if (-x /usr/afsws/bin/fs) then
          set fs=/usr/afsws/bin/fs
        else
          set fs=fs
        endif
	$fs la $HOME >& $tmp
	if ($status) then
		echo "unable to read acls on $HOME"
		exit
	endif
        cat $tmp | grep system:anyuser | grep rl >& /dev/null
	if ($status) then
           echo "Your home directory may need to be made world-readable for pvm"
	endif
	foreach d (~/pvm3 ~/pvm3/bin ~/pvm3/bin/$PVM_ARCH)
           $fs sa $d "system:anyuser" rl
	end
    endif
  endif
endif

#---------------------------------------------------------------------------*
# Phase 10: make sure .simrc file is present                                 *
#---------------------------------------------------------------------------*
if (! $skip) then
  if ($vrbose) then
	echo checking for simrc ...
  endif
  if (! -r $simrc && ! -r $HOME/$simrc ) then
	if ($altsimrc) then
		echo "can't find specified .simrc: $simrc"
		exit
	else
		echo "copying default .psimrc to ./.psimrc"
		cp $PGENESIS_INST/startup/.psimrc ./.psimrc
	endif
  else
	if ($vrbose) then 
		echo "found simrc"
	endif
  endif
endif

#---------------------------------------------------------------------------*
# Phase 11: check .rhosts set appropriately
#---------------------------------------------------------------------------*

# obsolete -- everyone should be using ssh-based PVM for security reasons!

#---------------------------------------------------------------------------*
# Phase 12: start pvm daemon
#---------------------------------------------------------------------------*

if (! $skip && $USE_PVM) then
  if ($vrbose) then
    echo "starting PVM daemon ($PVM)..."
  endif

  # catch interrupts, shut down pvmd and therefore all workers
  onintr shutdown
  # start the local pvm daemon, obtain configuration
  echo "conf" | $PVM $config >& $tmp
  if ($vrbose) then
	echo "version" | $PVM
  endif
  grep ' running' $tmp >& /dev/null
  if ($status) then
	echo "$PVM failed to start pvm daemon on" `hostname`
	rm -f $tmp
	exit
  endif

  # if multiple cpus, clean up ones not already running pvm daemon
  if (! $single) then
    if ($vrbose) then
      echo "checking if pvm already running on remote hosts..."
    endif
    foreach m ($remote)
      grep $m $tmp >& /dev/null
      if ($status) then
	if ($vrbose) then 
          echo "cleaning on $m..."
	endif
	ssh $m "rm -f /tmp/pvml.* /tmp/pvmd.*" >& /dev/null
      endif
    end
  endif

  # now try to start the daemon on all remote cpus
  if (! $single) then
    echo "add $remote" | $PVM >& $tmp
    foreach m ($remote)
      grep $m $tmp >& /dev/null
      if ($status) then
	echo "pvm daemon failed to start on $m, continuing..."
      endif
    end
  endif
  if ($vrbose) then
	echo "conf" | $PVM >& $tmp
	echo "PVM console reports daemon running on these hosts:"
	cat $tmp
  endif
  if ($vrbose) then
    echo resetting PVM daemons...
  endif
  echo "reset" | $PVM >& /dev/null
  rm -f $tmp
  sleep 1

endif

#---------------------------------------------------------------------------*
# Phase 13: fire up genesis
#---------------------------------------------------------------------------*

if (! $skip) then
  echo "starting pgenesis executable"
endif

if ($nox) then
  set exec=$PGENESIS_ROOT/bin/$PVM_ARCH/nxpgenesis
else
  set exec=$PGENESIS_ROOT/bin/$PVM_ARCH/pgenesis
endif

if ($USE_MPI) then
  if ( x$debug == x) then
    echo Going to run on $num_nodes nodes: $exec $nargv

# MPI_CMD
mpirun --verbose --mca plm_rsh_agent "/usr/bin/ssh" --host rauchbier -np $num_nodes nice -n10 $exec $nargv

  else
    echo Going to debug on $num_nodes nodes: $exec $nargv
# MPI_DEBUG_CMD
mpirun -gdb -np $num_nodes $exec $nargv
  endif

else if ($USE_PVM) then
  # run it under the debugger or not
  if ( x$debug == x || $debug == tty) then
	if ($vrbose) then
		echo $exec $nargv
	endif
	$exec $nargv
  else if ( $debug == gdb) then
	echo args are $nargv
	set TEMPCMD=$PGENESIS_TMP/debugger.$$
	set INITCMD=$PGENESIS_TMP/debuginit.$$
	echo dir "$PGENESIS_CODE"\:"$GENESIS_ROOT/src/sim"\:"$GENESIS_ROOT/src/sys"\:"$GENESIS_ROOT/src/ss"\:"$GENESIS_ROOT/src/shell"\:"$GENESIS_ROOT/src/newconn"  >> $INITCMD
	echo set args $nargv >> $INITCMD
	echo show args >> $INITCMD
	echo cd `pwd` >> $INITCMD
	echo "(defun rungdb ()" > $TEMPCMD
	echo "  (gdb "'"'gdb --command=$INITCMD $exec'"'"))" >> $TEMPCMD
	emacs -fn 9x15 -geometry 80x40 -T PGENESIS-MASTER@`hostname` -l $TEMPCMD -f rungdb >& /dev/null
	rm -f $TEMPCMD $INITCMD
  else
	echo "use these flags: $nargv"
	$debug $exec
  endif
else
  echo "Installation error: neither USE_MPI nor USE_PVM is set"
endif

#---------------------------------------------------------------------------*
# Phase 14: clean up
#---------------------------------------------------------------------------*

shutdown:

if ($USE_PVM && ! $skip  && ! $nohalt) then
  echo "halting PVM daemon and cleaning up ..."
  echo halt | $PVM >& /dev/null
  rm -f /tmp/pvml.* /tmp/pvmd.* >& /dev/null
  if (! $single) then
    foreach m ($remote)
      ssh $m "rm -f /tmp/pvml.* /tmp/pvmd.*" >& /dev/null
    end
  endif
  if ($PVM_ARCH == ALPHAMP || $PVM_ARCH == HPPAMP || \
      $PVM_ARCH == SUNMP || $PVM_ARCH == SGIMP64) then
    $PGENESIS_INST/bin/ipclean
  endif
endif