#!/bin/sh
#
# run_ens_fv3_wcoss2.sh
#   variant of run_ens_fv3.sh but for wcoss2
#
#   This program runs 80 copies of run_fv3_wcoss2_wrapper.sh 
#     which is a wrapper for run_fv3_wcoss2.sh
#
#  R&D version runs the 80 copies of the forecast system (with incremental updates)
#   by having the jobs run in background.  (Not all 80 at one time.)
#
#  This is a problem with wcoss2 because
#     1) background submissions of mpiexec fails
#     2) NCO does not allow background jobs in operations
#
#  Solution was suggested by GDIT which is used here.
#
#   makes first guess for all the ensemble members
# 
# THIS CODE IS SYSTEM DEPENDENT
# this is for a 1 node system (128 core per node)
#  running 4 forecast jobs per node

# v1.0  10/2023  W Ebisuzaki
set -x


export VERBOSE=YES

. $USHcore/setup_fatal.sh
export pgm="$0"

echo "$0: start `date`"

[ "$PARMcore" = '' ] && fatal "$0: undefined \$PARMcore"

. $PARMcore/config.model.sh	

[ "$NODES" -ne 1 ] && fatal  "$0: is configured for \$NODES=1"

export OMP_NUM_THREADS=$fg_threads
export OMP_STACKSIZE=$fg_OMP_STACKSIZE
export corespernode=$fg_corespernode

# try to keep job on same CCXes and not share CCXes
# Rome (Zen 2): CCX has 4 cores
# Milan (Zen 3): CCX has 8 cores
# Genoa (Zen 4): CCX has 8 cores

n=`expr $fg_threads \* $fg_procs`
echo "$0: n=$n procs needed"
# cpulist$i for model run
# 4 jobs node

for i in 0 1 2 3
do
  # not a problem if j== 0
  j=`expr $i \* 32` || err=1
  #  j=`expr $i \* 32 + 2`
  k=`expr $j + $n - 1`
  eval "cpu_list$i=`seq -s: $j $k`"
  eval "cpu_list0$i=`seq -s: $k -1 $j`"
done

# experimental options
# MPICH_SMP_SINGLE_COPY_SIZE
# MPICH_SMP_SINGLE_COPY_OFF

# set to any value, disable XMPMEM single copy method .. debug
# export MPICH_SMP_SINGLE_COPY_OFF='1'
# export MPICH_SMP_SINGLE_COPY_SIZE=500 .. chashed with mismatch with 10.5K message

# 10/29/2023
# export MPICH_SMP_SINGLE_COPY_SIZE=11000
# export MPICH_SMP_SINGLE_COPY_SIZE=500

# get memory high water mark
# export MPICH_MEMORY_REPORT=1

for mem in `seq  1 4 80`
do
  for i in 0 1 2 3
  do
    m=`expr $mem + $i`
    [ $m -le 9 ] && m=0$m
    eval "m$i=$m"
  done
  export niter=1
  echo "$0: Before first mpiexec: mem0$m0: epoch: `date +%s` `date`"
  mpiexec 2>&3 \
  -n 1 --cpu-bind none --mem-bind none ${USHcore}/run_fv3_wcoss2_wrapper.sh mem0$m0 "$cpu_list0" ${current_logdir}/run_fg_mem0$m0.out : \
  -n 1 --cpu-bind none --mem-bind none ${USHcore}/run_fv3_wcoss2_wrapper.sh mem0$m1 "$cpu_list1" ${current_logdir}/run_fg_mem0$m1.out : \
  -n 1 --cpu-bind none --mem-bind none ${USHcore}/run_fv3_wcoss2_wrapper.sh mem0$m2 "$cpu_list2" ${current_logdir}/run_fg_mem0$m2.out : \
  -n 1 --cpu-bind none --mem-bind none ${USHcore}/run_fv3_wcoss2_wrapper.sh mem0$m3 "$cpu_list3" ${current_logdir}/run_fg_mem0$m3.out
  err=$?
  echo "$0: After first mpiexec: mem0$m0: epoch: `date +%s` `date`"

  if [ $err -ne 0 ] ; then
     # failed, try with dt=dt/2
     export niter=2
     echo "$0: Before second mpiexec: mem0$m0: epoch: `date +%s` `date`"
     mpiexec 2>&3 \
     -n 1 --cpu-bind none --mem-bind none ${USHcore}/run_fv3_wcoss2_wrapper.sh mem0$m0 "$cpu_list0" ${current_logdir}/run_fg_mem0$m0.out : \
     -n 1 --cpu-bind none --mem-bind none ${USHcore}/run_fv3_wcoss2_wrapper.sh mem0$m1 "$cpu_list1" ${current_logdir}/run_fg_mem0$m1.out : \
     -n 1 --cpu-bind none --mem-bind none ${USHcore}/run_fv3_wcoss2_wrapper.sh mem0$m2 "$cpu_list2" ${current_logdir}/run_fg_mem0$m2.out : \
     -n 1 --cpu-bind none --mem-bind none ${USHcore}/run_fv3_wcoss2_wrapper.sh mem0$m3 "$cpu_list3" ${current_logdir}/run_fg_mem0$m3.out
     err=$?
     [ $err -ne 0 ] && fatal "$0: fv3 crash"
     echo "$0: After second mpiexec: mem0$m0: epoch: `date +%s` `date`"
  fi
done

echo "$0: all output files seem OK `date`"
exit 0
