Launch GEM coupled with NEMO

Necessary extra components

r.run_in_parallel (new mpi launcher)
u.make_node_map (needed by r.run_in_parallel)
${HOME}/CPL_EXEC must point to a directory NOT IN THE HOME FILESYSTEM

Modified components

Um_drive.ksh
- add sourcing gemdm_couplage_start.dot
- add sourcing /gemdm_couplage_end.dot
Um_model.ksh
- add '-instances gemdm -inorder -tag @@' to call to ${TASK_BIN}/r.mpirun

Modification to Um_drive.ksh (in generated model job)
##################################################################
# gemdm_couplage.dot gemdm_tiles gemdm_openmp instance_directory
# gemdm_tiles and gemdm_openmp from gem script
##################################################################
if [[ -r ${REPCFG}/gemdm_couplage_start.dot ]] ; then
export REPCFG=${REPCFG}
cd ${HOME}/CPL_EXEC || echo "===== WARNING: parallel TMPDIR in HOME filesystem ====="
. ${REPCFG}/gemdm_couplage_start.dot ${_cpus} ${_npeOMP} ${REPCFG}/instances
cd ${HOME}
fi
. . . . . . . . .
if [[ -r ${REPCFG}/gemdm_couplage_end.dot ]] ; then
. ${REPCFG}/gemdm_couplage_end.dot
fi

Before Um_lance / Chunk_lance

export SOUMET_EXTRAS='-q hb' (or appropriate queue)
export GEMDM_JOB_EXTRAS='-cpus Mx1' (M is model cpus + nemo cpus + 1)

scripts that must be found in configuration directory
- lance_nemo (to execute nemo)
- gemdm_couplage_start.dot (used to activate the compound MPI launch)
- gemdm_couplage_end.dot (things to do at termination)

##### a typical example for gemdm_couplage_start.dot #############
echo "configuration directory is : '${REPCFG}' "
s.use r.run_in_parallel as r.mpirun2
s.use r.run_in_parallel as r.mpirun
[[ -x ${REPCFG}/lance_nemo ]] || return
echo "########################################################"
echo "#        preparing for GEM/NEMO coupled run            #"
echo "########################################################"
set -x
#
# define instances control directory
#
export PARALLEL_INSTANCES_DIR="${3:-${REPCFG}/instances}"
#
# check that instances dir exists or can be created
mkdir -p $PARALLEL_INSTANCES_DIR && touch $PARALLEL_INSTANCES_DIR/.flag
[[ ! -w $PARALLEL_INSTANCES_DIR/.flag ]] && echo "instances directory $PARALLEL_INSTANCES_DIR not writable, aborting"
&& exit 1
rm -f $PARALLEL_INSTANCES_DIR/.flag
#
((JOB_CPUS=BATCH_MPI_CPUS*OMP_NUM_THREADS))
GEM_CPUS=${1}
GEM_OMP=${2}
#set value of NEMO_CPUS
((NEMO_CPUS=JOB_CPUS-GEM_CPUS*GEM_OMP-1))
#NEMO_CPUS=nnn
#
# master launcher in background nodemap may have to be adjusted
#
echo "node map : $(u.make_node_map 0 ${JOB_CPUS} ${GEM_CPUS}x${GEM_OMP} ${NEMO_CPUS}x1 1x1 )"
r.run_in_parallel -inorder -tag -instances gemdm nemo -npex 1 \
     -nodemap $(u.make_node_map 0 ${JOB_CPUS} ${GEM_CPUS}x${GEM_OMP} ${NEMO_CPUS}x1 1x1 ) \
     -pes $((GEM_CPUS+NEMO_CPUS+1)) -pgm ${REPCFG}/Single.Abs &
#
# launch nemo in background with appropriate number of cpus
# set RUN_IN_PARALLEL_EXTRAS here or use -instances nemo in nemo launcher
#
export RUN_IN_PARALLEL_EXTRAS='-instances nemo'
export BATCH_MPI_CPUS=${NEMO_CPUS}
export OMP_NUM_THREADS=1
${REPCFG}/lance_nemo &
unset RUN_IN_PARALLEL_EXTRAS
#
# back to gemdm script
#
export RUN_IN_PARALLEL_EXTRAS='-instances gemdm'
export BATCH_MPI_CPUS=${GEM_CPUS}
export OMP_NUM_THREADS=${GEM_OMP}
set +x

#####  a typical example for gemdm_couplage_end.dot #############

#wait for background tasks to terminate

wait

##### dummy example of lance_nemo
${TASK_BIN}/r.mpirun -pgm ${REPCFG}/Nemo.Abs -instances nemo