#!/bin/sh
#
# run $PGM using mpi, system dependant code
#
# usage: runmpi_parallel.sh  cpu_list program [args]
#   this is a variant of runmpi.sh
#   allows multiple mpi jobs per node
#   must give cpulist
#
# this is a special variant for wcoss2
#  wcoss2 requires special codes to run multiple mpi jobs at
#  one time.  Need to create 80 forecast runs of fv3.
#
#  want to execute run_fv3.sh 80 times
#  ideally run N instances at one time
#    ex 4 nodes, 3 fv3 runs can fit on one node
#    run 16 forecast runs at one time
#
#  limitation:
#     mpi job must run on current node
#     OpenMP is turned off (from example from GDIT)
#     given cpu_list
#       ex: 0:1:2:..:31
#     nprocs=determined from cpu_list
#     uses intel mpiexec (mpiexec.hydra)
#
# example use:
#
# 1.
#   mpiexec -pnn 2 -n 1 $0 1:2:3 "job1.x arglist" : -n 1 $0 4:5:6 "job2.x arglist"
#     runs job1.x on cpus 1:2:3   with OMP_NUM_THREADS=1
#     runs job2.x on cpus 4:5:6
#
# 2.
#   mpiexec -pnn 2 -n 1 $0 1:2:3 "job1.sh arglist" : -n 1 $0 4:5:6 "job2.sh arglist"
#    job1.sh contains
#       $0 1:2:3 "job1.x arglist"
#    job2.sh contains
#       $0 1:2:3 "job2.x arglist"
#
#  for a job that
#
# this is more convenient because you can search all the scripts by grep string *sh
#
# mpiexec_timeout: change to i_mpi_job_timeout for intel mpiexec
#
# v1.0:   8/2023 based on suggestions by GDIT,   Wesley Ebisuzaki

set -x
cpu_list=$1

. $USHcore/setup_fatal.sh
export pgm="$0"

if [ -z "$cpu_list" ] ; then
   echo "$0 cpu_list PGM"
   echo " runs mpiexec $PGM using cpu_list on current node"
   fatal "$0 cpu_list PGM"
fi

shift 1
PGM=$*
if [ "$PGM" = '' ] ; then
   echo "$0 no program to run"
   echo "  $0 cpu_list PGM"
   fatal "$0: missing cpulist"
fi

export OMP_NUM_THREADS=1
export OMP_PROC_BIND=false
this_node=${this_node:-`uname -n`}
# ulimit -s unlimited

export nprocs=`echo $cpu_list | tr -cd : | wc -c`
nprocs=`expr $nprocs + 1`
start_date=`date`
echo "$0: mpiexec-parallel pwd=`pwd` node= $this_node cpu_list=$cpu_list nprocs=$nprocs"
# echo "mpiexec -v --label --line-buffer -ppn $nprocs -n $nprocs --host $this_node --cpu-bind list:$cpu_list $PGM"
# mpiexec -v --label --line-buffer -ppn $nprocs -n $nprocs --host $this_node --cpu-bind list:$cpu_list $PGM
# mpiexec -v --label --line-buffer --verbose --universe-size 128 -n $nprocs --host $this_node --cpu-bind verbose,list:$cpu_list $PGM
# too slow mpiexec -v --label --line-buffer --verbose --universe-size 128 -n $nprocs --host $this_node $PGM
# too slow mpiexec -v --label --line-buffer --universe-size 128 -n $nprocs --host $this_node $PGM
mpiexec -v --label --line-buffer -ppn $nprocs -n $nprocs --host $this_node --cpu-bind list:$cpu_list --mem-bind local $PGM
err=$?
if [ $err -ne 0 ] ; then
  echo "$0: mpiexec-parallel start:$start_date finish:`date` return code=$err"
  echo "$0: return 0 because test"
  fatal "$0 error"
fi
exit 0
