Some clean up of variables from PR ls -l /scratch/scratch/zcapd47

src/multiply_module.f90

@@ -148,21 +148,18 @@ subroutine mat_mult(myid,a,lena,b,lenb,c,lenc,a_b_c,debug)
     integer :: n_cont(2),kpart_next,ind_partN,k_off(2)
     integer :: stat,ilen2(2),lenb_rem(2)
     ! Remote variables to be allocated
-    integer(integ),allocatable :: ibpart_rem(:,:)
+    integer(integ),allocatable :: ibpart_rem(:)
     type jagged_array_r
        real(double), allocatable :: values(:)
     end type jagged_array_r
     type(jagged_array_r) :: b_rem(2)
     ! Remote variables which will point to part_array
-     type jagged_pointer_array_i
-       integer(integ),pointer :: values(:)
-    end type jagged_pointer_array_i
-    type(jagged_pointer_array_i) :: nbnab_rem(2)
-    type(jagged_pointer_array_i) :: ibseq_rem(2)
-    type(jagged_pointer_array_i) :: ibind_rem(2)
-    type(jagged_pointer_array_i) :: ib_nd_acc_rem(2)
-    type(jagged_pointer_array_i) :: ibndimj_rem(2)
-    type(jagged_pointer_array_i) :: npxyz_rem(2)
+    integer(integ), dimension(:), pointer :: nbnab_rem
+    integer(integ), dimension(:), pointer :: ibseq_rem
+    integer(integ), dimension(:), pointer :: ibind_rem
+    integer(integ), dimension(:), pointer :: ib_nd_acc_rem
+    integer(integ), dimension(:), pointer :: ibndimj_rem
+    integer(integ), dimension(:), pointer :: npxyz_rem
     ! Arrays for remote variables to point to
     integer, target :: part_array(3*a_b_c%parts%mx_mem_grp+ &
          5*a_b_c%parts%mx_mem_grp*a_b_c%bmat(1)%mx_abs, 2)
@@ -172,7 +169,7 @@ subroutine mat_mult(myid,a,lena,b,lenb,c,lenc,a_b_c,debug)
     integer, dimension(MPI_STATUS_SIZE) :: mpi_stat
     integer, allocatable :: recv_part(:)
     real(double) :: t0,t1
-    integer :: request(2,2), index_rec, index_wait
+    integer :: request(2,2), index_rec, index_comp
 
     logical :: new_partition(2)
 
@@ -181,9 +178,8 @@ subroutine mat_mult(myid,a,lena,b,lenb,c,lenc,a_b_c,debug)
     call start_timer(tmr_std_allocation)
     if(iprint_mat>3.AND.myid==0) t0 = mtime()
     ! Allocate memory for the elements
-    allocate(ibpart_rem(a_b_c%parts%mx_mem_grp*a_b_c%bmat(1)%mx_abs,2),STAT=stat)
+    allocate(ibpart_rem(a_b_c%parts%mx_mem_grp*a_b_c%bmat(1)%mx_abs),STAT=stat)
     if(stat/=0) call cq_abort('mat_mult: error allocating ibpart_rem')
-    !allocate(atrans(a_b_c%amat(1)%length),STAT=stat)
     allocate(atrans(lena),STAT=stat)
     if(stat/=0) call cq_abort('mat_mult: error allocating atrans')
     allocate(recv_part(0:a_b_c%comms%inode),STAT=stat)
@@ -234,59 +230,58 @@ subroutine mat_mult(myid,a,lena,b,lenb,c,lenc,a_b_c,debug)
 
        ! These indices point to elements of all the 2-element vectors of the variables needed
        ! for the do_comms and m_kern_min/max calls. They alternate between the values of
-       ! (index_rec,index_wait)=(1,2) and (2,1) from iteration to iteration.
+       ! (index_rec,index_comp)=(1,2) and (2,1) from iteration to iteration.
        ! index_rec points to the values being received in the current iteration in do_comms,
-       ! and index_wait points to the values received in the previous iteration, thus computation
+       ! and index_comp points to the values received in the previous iteration, thus computation
        ! can start on them in m_kern_min/max
        ! These indices are also used to point to elements of the 2x2-element request() array,
        ! that contains the MPI request numbers for the non-blocking data receives. There are 2
        ! MPI_Irecv calls per call of do_comms, and request() keeps track of 2 sets of those calls,
        ! thus it's of size 2x2.
        ! request(:,index_rec) points to the requests from the current iteration MPI_Irecv,
-       ! and request(:,index_wait) points to the requests from the previous iteration, that have
+       ! and request(:,index_comp) points to the requests from the previous iteration, that have
        ! to complete in order for the computation to start (thus the MPI_Wait).
        index_rec = mod(kpart,2) + 1
-       index_wait = mod(kpart+1,2) + 1
+       index_comp = mod(kpart+1,2) + 1
 
        ! Check that previous partition data have been received before starting computation
        if (kpart.gt.2) then
-          if (request(1,index_wait).ne.MPI_REQUEST_NULL) &
-               call MPI_Wait(request(1,index_wait),MPI_STATUSES_IGNORE,ierr)
-          if (request(2,index_wait).ne.MPI_REQUEST_NULL) &
-               call MPI_Wait(request(2,index_wait),MPI_STATUSES_IGNORE,ierr)
+          if (request(1,index_comp).ne.MPI_REQUEST_NULL) &
+               call MPI_Wait(request(1,index_comp),MPI_STATUSES_IGNORE,ierr)
+          if (request(2,index_comp).ne.MPI_REQUEST_NULL) &
+               call MPI_Wait(request(2,index_comp),MPI_STATUSES_IGNORE,ierr)
        end if
 
        ! If that previous partition was a periodic one, copy over arrays from previous index
-       if(.not.new_partition(index_wait)) then
-          part_array(:,index_wait) = part_array(:,index_rec)
-          n_cont(index_wait) = n_cont(index_rec)
-          ilen2(index_wait) = ilen2(index_rec)
-          b_rem(index_wait) = b_rem(index_rec)
-          lenb_rem(index_wait) = lenb_rem(index_rec)
+       if(.not.new_partition(index_comp)) then
+          part_array(:,index_comp) = part_array(:,index_rec)
+          n_cont(index_comp) = n_cont(index_rec)
+          ilen2(index_comp) = ilen2(index_rec)
+          b_rem(index_comp) = b_rem(index_rec)
+          lenb_rem(index_comp) = lenb_rem(index_rec)
        end if
 
         ! Now point the _rem variables at the appropriate parts of
         ! the array where we have received the data
         offset = 0
-        nbnab_rem(index_wait)%values => part_array(offset+1:offset+n_cont(index_wait),index_wait)
-        offset = offset+n_cont(index_wait)
-        ibind_rem(index_wait)%values => part_array(offset+1:offset+n_cont(index_wait),index_wait)
-        offset = offset+n_cont(index_wait)
-        ib_nd_acc_rem(index_wait)%values => part_array(offset+1:offset+n_cont(index_wait),index_wait)
-        offset = offset+n_cont(index_wait)
-        ibseq_rem(index_wait)%values => part_array(offset+1:offset+ilen2(index_wait),index_wait)
-        offset = offset+ilen2(index_wait)
-        npxyz_rem(index_wait)%values => part_array(offset+1:offset+3*ilen2(index_wait),index_wait)
-        offset = offset+3*ilen2(index_wait)
-        ibndimj_rem(index_wait)%values => part_array(offset+1:offset+ilen2(index_wait),index_wait)
-        if(offset+ilen2(index_wait)>3*a_b_c%parts%mx_mem_grp+ &
+        nbnab_rem => part_array(offset+1:offset+n_cont(index_comp),index_comp)
+        offset = offset+n_cont(index_comp)
+        ibind_rem => part_array(offset+1:offset+n_cont(index_comp),index_comp)
+        offset = offset+n_cont(index_comp)
+        ib_nd_acc_rem => part_array(offset+1:offset+n_cont(index_comp),index_comp)
+        offset = offset+n_cont(index_comp)
+        ibseq_rem => part_array(offset+1:offset+ilen2(index_comp),index_comp)
+        offset = offset+ilen2(index_comp)
+        npxyz_rem => part_array(offset+1:offset+3*ilen2(index_comp),index_comp)
+        offset = offset+3*ilen2(index_comp)
+        ibndimj_rem => part_array(offset+1:offset+ilen2(index_comp),index_comp)
+        if(offset+ilen2(index_comp)>3*a_b_c%parts%mx_mem_grp+ &
              5*a_b_c%parts%mx_mem_grp*a_b_c%bmat(1)%mx_abs) then
            call cq_abort('mat_mult: error pointing to part_array ',kpart-1)
         end if
         ! Create ibpart_rem
-        call end_part_comms(myid,n_cont(index_wait),nbnab_rem(index_wait)%values, &
-             ibind_rem(index_wait)%values,npxyz_rem(index_wait)%values,&
-             ibpart_rem(:,index_wait),ncover_yz,a_b_c%gcs%ncoverz)
+        call end_part_comms(myid,n_cont(index_comp),nbnab_rem, &
+             ibind_rem,npxyz_rem,ibpart_rem,ncover_yz,a_b_c%gcs%ncoverz)
 
         ! Receive the data from the current partition - non-blocking
         if (kpart.lt.a_b_c%ahalo%np_in_halo+1) then
@@ -303,17 +298,17 @@ subroutine mat_mult(myid,a,lena,b,lenb,c,lenc,a_b_c,debug)
 
        ! Call the computation kernel on the previous partition
        if(a_b_c%mult_type.eq.1) then  ! C is full mult
-          call m_kern_max( k_off(index_wait),kpart-1,ib_nd_acc_rem(index_wait)%values, ibind_rem(index_wait)%values, &
-               nbnab_rem(index_wait)%values,ibpart_rem(:,index_wait),ibseq_rem(index_wait)%values, &
-               ibndimj_rem(index_wait)%values, atrans,b_rem(index_wait)%values,c,a_b_c%ahalo,a_b_c%chalo, &
+          call m_kern_max( k_off(index_comp),kpart-1,ib_nd_acc_rem, ibind_rem, &
+               nbnab_rem,ibpart_rem,ibseq_rem, &
+               ibndimj_rem, atrans,b_rem(index_comp)%values,c,a_b_c%ahalo,a_b_c%chalo, &
                a_b_c%ltrans,a_b_c%bmat(1)%mx_abs,a_b_c%parts%mx_mem_grp, &
-               a_b_c%prim%mx_iprim, lena, lenb_rem(index_wait), lenc)
+               a_b_c%prim%mx_iprim, lena, lenb_rem(index_comp), lenc)
        else if(a_b_c%mult_type.eq.2) then ! A is partial mult
-          call m_kern_min( k_off(index_wait),kpart-1,ib_nd_acc_rem(index_wait)%values, ibind_rem(index_wait)%values, &
-               nbnab_rem(index_wait)%values,ibpart_rem(:,index_wait),ibseq_rem(index_wait)%values, &
-               ibndimj_rem(index_wait)%values, atrans,b_rem(index_wait)%values,c,a_b_c%ahalo,a_b_c%chalo, &
+          call m_kern_min( k_off(index_comp),kpart-1,ib_nd_acc_rem, ibind_rem, &
+               nbnab_rem,ibpart_rem,ibseq_rem, &
+               ibndimj_rem, atrans,b_rem(index_comp)%values,c,a_b_c%ahalo,a_b_c%chalo, &
                a_b_c%ltrans,a_b_c%bmat(1)%mx_abs,a_b_c%parts%mx_mem_grp, &
-               a_b_c%prim%mx_iprim, lena, lenb_rem(index_wait), lenc)
+               a_b_c%prim%mx_iprim, lena, lenb_rem(index_comp), lenc)
        end if
        !$omp barrier
     end do main_loop    
@@ -586,7 +581,7 @@ subroutine do_comms(k_off, kpart, part_array, n_cont, ilen2, a_b_c, b, recv_part
     integer, intent(in) :: kpart
     type(matrix_mult), intent(in) :: a_b_c
     real(double), intent(in) :: b(:)
-    integer, allocatable, dimension(:), intent(inout) :: recv_part
+    integer, dimension(:), intent(inout) :: recv_part
     real(double), allocatable, intent(inout) :: b_rem(:)
     integer, intent(out) :: lenb_rem
     integer, intent(in) :: myid, ncover_yz
@@ -712,23 +707,23 @@ subroutine prefetch(this_part,ahalo,a_b_c,bmat,&
             ind_part,b,myid)
     else ! Else fetch the data
        ilen2 = a_b_c%ilen2rec(ipart,nnode)
-       if(.not.do_nonb_local) then ! Use blocking receive
-          call Mquest_get( prim%mx_ngonn, &
+       if(do_nonb_local) then ! Use non-blocking receive
+          if (.not.present(request)) call cq_abort('Need to provide MPI request argument for non-blocking receive.')
+          call Mquest_get_nonb( prim%mx_ngonn, &
                a_b_c%ilen2rec(ipart,nnode),&
                a_b_c%ilen3rec(ipart,nnode),&
                n_cont,inode,ipart,myid,&
                bind_rem,b_rem,lenb_rem,bind,&
                a_b_c%istart(ipart,nnode), &
-               bmat(1)%mx_abs,parts%mx_mem_grp,tag)
-       else ! Use non-blocking receive
-          if (.not.present(request)) call cq_abort('Need to provide MPI request argument for non-blocking receive.')
-          call Mquest_get_nonb( prim%mx_ngonn, &
+               bmat(1)%mx_abs,parts%mx_mem_grp,tag,request)
+       else ! Use blocking receive
+          call Mquest_get( prim%mx_ngonn, &
                a_b_c%ilen2rec(ipart,nnode),&
                a_b_c%ilen3rec(ipart,nnode),&
                n_cont,inode,ipart,myid,&
                bind_rem,b_rem,lenb_rem,bind,&
                a_b_c%istart(ipart,nnode), &
-               bmat(1)%mx_abs,parts%mx_mem_grp,tag,request)
+               bmat(1)%mx_abs,parts%mx_mem_grp,tag)
        end if
     end if
     return