Patch for more verbose debug on RegCM SVN 6916.

RegCM-SVN6916_more-io-debug.patch

diff -Naur -x '*.o' -x '*.mod' -x '*.a' -x 'config.*' -x configure -x 'aclocal*' -x depcomp -x external -x .deps -x myabort__genmod.f90 -x missing -x '*Proc' -x libtool -x '*sh' -x 'stamp*' -x 'Makefile*' RegCM-SVN6916_orig/Main/mod_ncio.F90 RegCM-SVN6916_patch/Main/mod_ncio.F90
--- RegCM-SVN6916_orig/Main/mod_ncio.F90	2017-11-28 12:11:10.000000000 +0100
+++ RegCM-SVN6916_patch/Main/mod_ncio.F90	2019-03-27 17:07:56.704690606 +0100
@@ -28,6 +28,7 @@
   use mod_memutil
   use mod_nchelper
   use mod_domain
+  use mod_service
 
   implicit none
 
@@ -91,6 +92,12 @@
     logical :: has_dhlake = .true.
     logical :: lerror
 
+#ifdef DEBUG
+    character(len=dbgslen) :: subroutine_name = 'read_domaininfo'
+    integer(ik4) , save :: idindx = 0
+    call time_begin(subroutine_name,idindx)
+#endif
+
     dname = trim(dirter)//pthsep//trim(domname)//'_DOMAIN000.nc'
     if ( myid == italk ) then
       write(stdout,*) 'Reading Domain file : ', trim(dname)
@@ -299,6 +306,10 @@
         call getmem2d(tempwtop,jce1,jce2,ice1,ice2,'read_domain:tempwtop')
       end if
     end if
+
+#ifdef DEBUG
+    call time_end(subroutine_name,idindx)
+#endif
   end subroutine read_domain_info
 
   subroutine read_subdomain_info(ht,lnd,tex,mask,xlat,xlon,hlake)
@@ -318,6 +329,12 @@
     logical :: has_dhlake = .true.
     character(len=3) :: sbstring
 
+#ifdef DEBUG
+    character(len=dbgslen) :: subroutine_name = 'read_subdomain_info'
+    integer(ik4) , save :: idindx = 0
+    call time_begin(subroutine_name,idindx)
+#endif
+
     write (sbstring,'(i0.3)') nsg
     dname = trim(dirter)//pthsep//trim(domname)//'_DOMAIN'//sbstring//'.nc'
     if ( myid == italk ) then
@@ -420,6 +437,11 @@
         end if
       end if
     end if
+
+#ifdef DEBUG
+    call time_end(subroutine_name,idindx)
+#endif
+
   end subroutine read_subdomain_info
 
   integer function icbc_search(idate)
@@ -480,9 +502,20 @@
     real(rkx) , dimension(:) , allocatable :: icbc_nctime
     character(len=64) :: icbc_timeunits , icbc_timecal
     character(len=256) :: icbcname
+
+#ifdef DEBUG
+    character(len=dbgslen) :: subroutine_name = 'open_icbc'
+    integer(ik4) , save :: idindx = 0
+    call time_begin(subroutine_name,idindx)
+#endif
+
     if ( .not. do_parallel_netcdf_in .and. myid /= iocpu ) then
+#ifdef DEBUG
+    call time_end(subroutine_name,idindx)
+#endif
       return
     end if
+
     call close_icbc
     write (ctime, '(a)') tochar10(idate)
     icbcname = trim(dirglob)//pthsep//trim(domname)// &
@@ -563,6 +596,11 @@
       allocate(rspace2(jx,iy))
       allocate(rspace3(jx,iy,kz))
     end if
+
+#ifdef DEBUG
+    call time_end(subroutine_name,idindx)
+#endif
+
   end subroutine open_icbc
 
   subroutine open_som
@@ -601,6 +639,12 @@
     integer(ik4) :: i , j , k
     real(rkx) :: told , pold , rhold , satvp , tnew , pnew
 
+#ifdef DEBUG
+    character(len=dbgslen) :: subroutine_name = 'read_icbc'
+    integer(ik4) , save :: idindx = 0
+    call time_begin(subroutine_name,idindx)
+#endif
+
     if ( do_parallel_netcdf_in ) then
       istart(1) = jde1
       istart(2) = ide1
@@ -846,6 +890,10 @@
       end if
     end if
 
+#ifdef DEBUG
+    call time_end(subroutine_name,idindx)
+#endif
+
     contains
 
 #include <pfesat.inc>
@@ -895,6 +943,15 @@
 
   subroutine close_icbc
     implicit none
+
+! Instrumenting this subrouting breaks at runtime with error:
+!   Fatal in file: mod_service.F90 ..., different trees on different pe!
+!#ifdef DEBUG
+!    character(len=dbgslen) :: subroutine_name = 'close_icbc'
+!    integer(ik4) , save :: idindx = 0
+!    call time_begin(subroutine_name,idindx)
+!#endif
+
     if (ibcin >= 0) then
       istatus = nf90_close(ibcin)
       call check_ok(__FILE__,__LINE__,'Error Close ICBC file','ICBC FILE')
@@ -903,6 +960,11 @@
     end if
     if ( associated(rspace2) ) deallocate(rspace2)
     if ( associated(rspace3) ) deallocate(rspace3)
+
+!#ifdef DEBUG
+!    call time_end(subroutine_name,idindx)
+!#endif
+
   end subroutine close_icbc
 
   subroutine close_som
diff -Naur -x '*.o' -x '*.mod' -x '*.a' -x 'config.*' -x configure -x 'aclocal*' -x depcomp -x external -x .deps -x myabort__genmod.f90 -x missing -x '*Proc' -x libtool -x '*sh' -x 'stamp*' -x 'Makefile*' RegCM-SVN6916_orig/Main/mod_savefile.F90 RegCM-SVN6916_patch/Main/mod_savefile.F90
--- RegCM-SVN6916_orig/Main/mod_savefile.F90	2018-10-11 13:43:23.000000000 +0200
+++ RegCM-SVN6916_patch/Main/mod_savefile.F90	2019-03-25 12:50:52.101973094 +0100
@@ -32,6 +32,7 @@
   use mod_che_interface
   use mod_che_mppio
   use mod_massck
+  use mod_service
 
   implicit none
 
@@ -189,6 +190,12 @@
   subroutine allocate_mod_savefile
     implicit none
 
+#ifdef DEBUG
+    character(len=dbgslen) :: subroutine_name = 'allocate_mod_savefile'
+    integer(ik4) , save :: idindx = 0
+    call time_begin(subroutine_name,idindx)
+#endif
+
     if ( myid == iocpu ) then
       call getmem3d(atm1_u_io,jdot1,jdot2,idot1,idot2,1,kz,'atm1_u_io')
       call getmem3d(atm1_v_io,jdot1,jdot2,idot1,idot2,1,kz,'atm1_v_io')
@@ -348,6 +355,11 @@
       call getmem3d(lwdifalb_io,1,nnsg,jcross1,jcross2, &
                                        icross1,icross2,'lwdifalb')
     endif
+
+#ifdef DEBUG
+    call time_end(subroutine_name,idindx)
+#endif
+
   end subroutine allocate_mod_savefile
 
   subroutine read_savefile(idate)
@@ -361,6 +373,12 @@
     character(256) :: ffin
     character(32) :: fbname
 
+#ifdef DEBUG
+    character(len=dbgslen) :: subroutine_name = 'read_savefile'
+    integer(ik4) , save :: idindx = 0
+    call time_begin(subroutine_name,idindx)
+#endif
+
     if ( myid == iocpu ) then
       write (fbname, '(a,a)') 'SAV.', trim(tochar10(idate))
       ffin = trim(dirout)//pthsep//trim(domname)//'_'//trim(fbname)//'.nc'
@@ -655,6 +673,11 @@
       ncstatus = nf90_close(ncid)
       call check_ok(__FILE__,__LINE__,'Cannot close savefile '//trim(ffin))
     end if
+
+#ifdef DEBUG
+    call time_end(subroutine_name,idindx)
+#endif
+
   end subroutine read_savefile
 
   subroutine write_savefile(idate)
@@ -671,6 +694,12 @@
     integer(ik4) :: ioff
 #endif
 
+#ifdef DEBUG
+    character(len=dbgslen) :: subroutine_name = 'write_savefile'
+    integer(ik4) , save :: idindx = 0
+    call time_begin(subroutine_name,idindx)
+#endif
+
     if ( myid == iocpu ) then
       write (fbname, '(a,a)') 'SAV.', trim(tochar10(idate))
       ffout = trim(dirout)//pthsep//trim(domname)//'_'//trim(fbname)//'.nc'
@@ -1099,6 +1128,11 @@
     if ( myid == iocpu ) then
       write(stdout,*) 'SAV variables written at ', rcmtimer%str( )
     end if
+
+#ifdef DEBUG
+    call time_end(subroutine_name,idindx)
+#endif
+
   end subroutine write_savefile
 
   subroutine check_ok(f,l,m1)
diff -Naur -x '*.o' -x '*.mod' -x '*.a' -x 'config.*' -x configure -x 'aclocal*' -x depcomp -x external -x .deps -x myabort__genmod.f90 -x missing -x '*Proc' -x libtool -x '*sh' -x 'stamp*' -x 'Makefile*' RegCM-SVN6916_orig/Main/mpplib/mod_ncout.F90 RegCM-SVN6916_patch/Main/mpplib/mod_ncout.F90
--- RegCM-SVN6916_orig/Main/mpplib/mod_ncout.F90	2018-09-25 15:57:57.000000000 +0200
+++ RegCM-SVN6916_patch/Main/mpplib/mod_ncout.F90	2019-03-26 11:19:52.580112037 +0100
@@ -488,6 +488,12 @@
     integer(ik4) :: global_out_jstart , global_out_jend
     integer(ik4) :: global_out_istart , global_out_iend
 
+#ifdef DEBUG
+    character(len=dbgslen) :: subroutine_name = 'init_output_streams'
+    integer(ik4) , save :: idindx = 0
+    call time_begin(subroutine_name,idindx)
+#endif
+
     parallel_out = lparallel
 
     global_out_jstart = 1
@@ -2574,6 +2580,10 @@
       end if
     end if
 
+#ifdef DEBUG
+    call time_end(subroutine_name,idindx)
+#endif
+
   end subroutine init_output_streams
 
   integer(ik4) function countvars(eflags,ntot)
@@ -2598,6 +2608,12 @@
     integer(ik4) :: i , j , ivar
     real(rkx) :: dummy
 
+#ifdef DEBUG
+    character(len=dbgslen) :: subroutine_name = 'newoutfiles'
+    integer(ik4) , save :: idindx = 0
+    call time_begin(subroutine_name,idindx)
+#endif
+
     if ( .not. parallel_out .and. myid /= iocpu ) then
       stream_loop: &
       do i = 1 , maxstreams
@@ -2616,6 +2632,10 @@
           end do var_loop
         end do file_loop
       end do stream_loop
+
+#ifdef DEBUG
+      call time_end(subroutine_name,idindx)
+#endif
       return
     end if
 
@@ -3359,6 +3379,10 @@
         end do var_loop_par
       end do file_loop_par
     end do stream_loop_par
+
+#ifdef DEBUG
+    call time_end(subroutine_name,idindx)
+#endif
   end subroutine newoutfiles
 
   subroutine setup_common_vars(vsize,var,xlon,xlat,topo,mask,ps,ps0)
@@ -3366,6 +3390,13 @@
     type(varspan) , intent(in) :: vsize
     type(ncvariable2d_mixed) , dimension(:) , intent(inout) :: var
     integer(ik4), intent(in) :: xlon , xlat , topo , mask , ps , ps0
+
+#ifdef DEBUG
+    character(len=dbgslen) :: subroutine_name = 'newoutfiles'
+    integer(ik4) , save :: idindx = 0
+    call time_begin(subroutine_name,idindx)
+#endif
+
     if ( associated(xlon_out) ) then
       call setup_var(var,xlon,vsize,'xlon','degrees_east', &
         'Longitude on Cross Points','longitude',lgetspace=.false.)
@@ -3418,6 +3449,11 @@
         p0_out => var(ps0)%rval
       end if
     end if
+
+#ifdef DEBUG
+    call time_end(subroutine_name,idindx)
+#endif
+
   end subroutine setup_common_vars
 
   subroutine setup_var_2d(var,ivar,vsize,vname,vunit,long_name,standard_name, &
@@ -3553,6 +3589,13 @@
  subroutine dispose_output_streams
     implicit none
     integer(ik4) :: nstream , nfile
+
+#ifdef DEBUG
+    character(len=dbgslen) :: subroutine_name = 'dispose_output_streams'
+    integer(ik4) , save :: idindx = 0
+    call time_begin(subroutine_name,idindx)
+#endif
+
     if ( associated(v2dvar_atm) ) deallocate(v2dvar_atm)
     if ( associated(v3dvar_atm) ) deallocate(v3dvar_atm)
     if ( associated(v2dvar_srf) ) deallocate(v2dvar_srf)
@@ -3582,6 +3625,11 @@
       end if
     end do
     deallocate(outstream)
+
+#ifdef DEBUG
+    call time_end(subroutine_name,idindx)
+#endif
+
   end subroutine dispose_output_streams
 
   subroutine write_record_output_stream(istream,idate,ifile)
@@ -3598,6 +3646,12 @@
     class(ncvariable_standard) , pointer :: vp
     integer(ik4) :: ivar , jfile
 
+#ifdef DEBUG
+    character(len=dbgslen) :: subroutine_name = 'write_record_output_stream'
+    integer(ik4) , save :: idindx = 0
+    call time_begin(subroutine_name,idindx)
+#endif
+
     if ( .not. parallel_out .and. myid /= iocpu ) then
       do ivar = 1 , outstream(istream)%nvar
         vp => outstream(istream)%ncvars%vlist(ivar)%vp
@@ -3617,6 +3671,10 @@
         end select
       end do
 
+#ifdef DEBUG
+    call time_end(subroutine_name,idindx)
+#endif
+
       ! If not parallel output, only the master proc writes output files
       return
 
@@ -3626,6 +3684,11 @@
     if ( present(ifile) ) jfile = ifile
     if ( jfile < 1 .or. jfile > outstream(istream)%nfiles ) then
       write (stderr,*) 'No such file in stream ',istream,' : ', ifile
+
+#ifdef DEBUG
+    call time_end(subroutine_name,idindx)
+#endif
+
       return
     end if
 
@@ -3762,9 +3825,12 @@
             cycle
         end select
       end if
-
     end do
 
+#ifdef DEBUG
+    call time_end(subroutine_name,idindx)
+#endif
+
   end subroutine write_record_output_stream
 
   subroutine writevar2d_output_stream(istream,vp,ifile)
@@ -3776,6 +3842,12 @@
     real(rkx) , pointer , dimension(:,:) :: pnt2d => null( )
     integer(ik4) :: jfile
 
+#ifdef DEBUG
+    character(len=dbgslen) :: subroutine_name = 'writevar2d_output_stream'
+    integer(ik4) , save :: idindx = 0
+    call time_begin(subroutine_name,idindx)
+#endif
+
     if ( .not. parallel_out .and. myid /= iocpu ) then
       call grid_collect(vp%rval,pnt2d,vp%j1,vp%j2,vp%i1,vp%i2)
       ! If not parallel output, only the master proc writes output files
@@ -3826,6 +3898,10 @@
       vp%i2 = outstream(istream)%il2
     end if
 
+#ifdef DEBUG
+    call time_end(subroutine_name,idindx)
+#endif
+
   end subroutine writevar2d_output_stream
 
   subroutine writevar3d_output_stream(istream,vp,ifile)
@@ -3837,9 +3913,20 @@
     real(rkx) , pointer , dimension(:,:,:) :: pnt3d => null( )
     integer(ik4) :: jfile
 
+#ifdef DEBUG
+    character(len=dbgslen) :: subroutine_name = 'writevar3d_output_stream'
+    integer(ik4) , save :: idindx = 0
+    call time_begin(subroutine_name,idindx)
+#endif
+
     if ( .not. parallel_out .and. myid /= iocpu ) then
       call grid_collect(vp%rval,pnt3d,vp%j1,vp%j2,vp%i1,vp%i2,vp%k1,vp%k2)
       ! If not parallel output, only the master proc writes output files
+
+#ifdef DEBUG
+    call time_end(subroutine_name,idindx)
+#endif
+
       return
     end if
 
@@ -3847,6 +3934,11 @@
     if ( present(ifile) ) jfile = ifile
     if ( jfile < 1 .or. jfile > outstream(istream)%nfiles ) then
       write (stderr,*) 'No such file in stream ',istream,' : ', ifile
+
+#ifdef DEBUG
+    call time_end(subroutine_name,idindx)
+#endif
+
       return
     end if
 
@@ -3886,6 +3978,11 @@
       vp%i1 = outstream(istream)%il1
       vp%i2 = outstream(istream)%il2
     end if
+
+#ifdef DEBUG
+    call time_end(subroutine_name,idindx)
+#endif
+
   end subroutine writevar3d_output_stream
 
   subroutine writevar4d_output_stream(istream,vp,ifile)
@@ -3897,9 +3994,20 @@
     real(rkx) , pointer , dimension(:,:,:,:) :: pnt4d => null( )
     integer(ik4) :: jfile
 
+#ifdef DEBUG
+    character(len=dbgslen) :: subroutine_name = 'writevar4d_output_stream'
+    integer(ik4) , save :: idindx = 0
+    call time_begin(subroutine_name,idindx)
+#endif
+
     if ( .not. parallel_out .and. myid /= iocpu ) then
       call grid_collect(vp%rval,pnt4d,vp%j1,vp%j2, &
                         vp%i1,vp%i2,vp%k1,vp%k2,vp%n1,vp%n2)
+
+#ifdef DEBUG
+    call time_end(subroutine_name,idindx)
+#endif
+
       ! If not parallel output, only the master proc writes output files
       return
     end if
@@ -3908,6 +4016,11 @@
     if ( present(ifile) ) jfile = ifile
     if ( jfile < 1 .or. jfile > outstream(istream)%nfiles ) then
       write (stderr,*) 'No such file in stream ',istream,' : ', ifile
+
+#ifdef DEBUG
+    call time_end(subroutine_name,idindx)
+#endif
+
       return
     end if
 
@@ -3944,6 +4057,10 @@
       vp%i1 = outstream(istream)%il1
       vp%i2 = outstream(istream)%il2
     end if
+
+#ifdef DEBUG
+    call time_end(subroutine_name,idindx)
+#endif
   end subroutine writevar4d_output_stream
 
 end module mod_ncout

If compiling manually (without spack), you need to edit also configure.ac, and change the flags for the intel compilation under debug to -O2 or -O3 (-O0 breaks compilation because it doesn't find some exchange_lr_ functions).