Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

initial commit for multi FESOM group iceberg loop parallelization (MPI based) #364

Open
wants to merge 2 commits into
base: 2.1-awiesm-2.1
Choose a base branch
from
Open

initial commit for multi FESOM group iceberg loop parallelization (MPI based) #364

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6de0d4c
buoy_props array bound adjusted
Aug 4, 2022
5e43ff0
initial commit for multi FESOM group iceberg loop parallelization (MP…
Aug 4, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 65 additions & 29 deletions src/cpl_driver.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ module cpl_driver
!
use mod_oasis ! oasis module
!---wiso-code
use g_config, only : dt, lwiso ! add lwiso switch
use g_config, only : dt, lwiso, & ! add lwiso switch
num_fesom_groups ! kh 03.12.21
!---wiso-code-end
use o_param, only : rad
use g_PARSUP
Expand Down Expand Up @@ -102,7 +103,8 @@ module cpl_driver

contains

subroutine cpl_oasis3mct_init( localCommunicator )
! kh 02.12.21
subroutine cpl_oasis3mct_init(localCommunicator, num_fesom_groups)
implicit none
save

Expand All @@ -114,6 +116,9 @@ subroutine cpl_oasis3mct_init( localCommunicator )
! Arguments
!
integer, intent(OUT) :: localCommunicator

! kh 02.12.21
integer, intent(inout) :: num_fesom_groups
!
! Local declarations
!
Expand All @@ -135,7 +140,9 @@ subroutine cpl_oasis3mct_init( localCommunicator )
!------------------------------------------------------------------
! 1st Initialize the OASIS3-MCT coupling system for the application
!------------------------------------------------------------------
CALL oasis_init_comp(comp_id, comp_name, ierror )

! kh 02.12.21
CALL oasis_init_comp(comp_id, comp_name, ierror, num_program_groups = num_fesom_groups)
IF (ierror /= 0) THEN
CALL oasis_abort(comp_id, 'cpl_oasis3mct_init', 'Init_comp failed.')
ENDIF
Expand All @@ -146,7 +153,8 @@ subroutine cpl_oasis3mct_init( localCommunicator )
CALL oasis_abort(comp_id, 'cpl_oasis3mct_init', 'comm_rank failed.')
ENDIF

CALL oasis_get_localcomm( localCommunicator, ierror )
! kh 02.12.21
CALL oasis_get_localcomm_all_groups( localCommunicator, ierror )
IF (ierror /= 0) THEN
CALL oasis_abort(comp_id, 'cpl_oasis3mct_init', 'get_local_comm failed.')
ENDIF
Expand Down Expand Up @@ -362,30 +370,31 @@ subroutine cpl_oasis3mct_define_unstr(mesh)
print *, 'FESOM after Barrier'
endif

if (mype .eq. localroot) then
print *, 'FESOM before start_grids_writing'
CALL oasis_start_grids_writing(il_flag)
IF (il_flag .NE. 0) THEN
! kh 30.11.21
if(my_fesom_group == 0) then
if (mype .eq. localroot) then
print *, 'FESOM before start_grids_writing'
CALL oasis_start_grids_writing(il_flag)
IF (il_flag .NE. 0) THEN

print *, 'FESOM before write grid'
CALL oasis_write_grid (grid_name, number_of_all_points, 1, all_x_coords(:,:), all_y_coords(:,:))
print *, 'FESOM before write grid'
CALL oasis_write_grid (grid_name, number_of_all_points, 1, all_x_coords(:,:), all_y_coords(:,:))

ALLOCATE(unstr_mask(number_of_all_points, 1))
unstr_mask=0
print *, 'FESOM before write mask'
CALL oasis_write_mask(grid_name, number_of_all_points, 1, unstr_mask)
DEALLOCATE(unstr_mask)
ALLOCATE(unstr_mask(number_of_all_points, 1))
unstr_mask=0
print *, 'FESOM before write mask'
CALL oasis_write_mask(grid_name, number_of_all_points, 1, unstr_mask)
DEALLOCATE(unstr_mask)

print *, 'FESOM before write area'
CALL oasis_write_area(grid_name, number_of_all_points, 1, all_area)

end if
print *, 'FESOM before terminate_grids_writing'
call oasis_terminate_grids_writing()
print *, 'FESOM after terminate_grids_writing'
endif !localroot

print *, 'FESOM before write area'
CALL oasis_write_area(grid_name, number_of_all_points, 1, all_area)

end if
print *, 'FESOM before terminate_grids_writing'
call oasis_terminate_grids_writing()
print *, 'FESOM after terminate_grids_writing'
endif !localroot
end if ! (my_fesom_group == 0) then

DEALLOCATE(all_x_coords, all_y_coords, my_x_coords, my_y_coords)
!------------------------------------------------------------------
Expand Down Expand Up @@ -636,28 +645,55 @@ subroutine cpl_oasis3mct_recv(ind, data_array, action)
!
!--------------------------------------------------------------------
!
t1=MPI_Wtime()
t1=MPI_Wtime()
action=.false.
!
! receive data from OASIS3-MCT on local root
!
#ifdef VERBOSE
if (mype==0) then
print *, 'oasis_get: ', cpl_recv(ind)
endif
endif
#endif

call oasis_get(recv_id(ind), seconds_til_now, exfld,info)
if(my_fesom_group == 0) then

! kh 06.12.21 the coupling is in principle as it was before, i.e. the fesom processes - in group 0 - receive their data from echam
call oasis_get(recv_id(ind), seconds_til_now, exfld,info)
else

! kh 06.12.21 defensive: assignment statement "action=(info==3 ..." below is "don't care" in this case, because the actual value for action
! is received via MPI_Bcast anway
info = 0

end if
t2=MPI_Wtime()

!
! FESOM's interpolation routine interpolates structured
! VarStrLoc coming from OASIS3MCT to local unstructured data_array
! and delivered back to FESOM.
action=(info==3 .OR. info==10 .OR. info==11 .OR. info==12 .OR. info==13)

! kh 03.12.21
if(num_fesom_groups > 1) then
call MPI_Bcast(action, 1, MPI_LOGICAL, 0, MPI_COMM_FESOM_SAME_RANK_IN_GROUPS, MPIerr)
end if

if (action) then
data_array(1:myDim_nod2d) = exfld

! kh 03.12.21
if(my_fesom_group == 0) then
data_array(1:myDim_nod2d) = exfld
end if

! kh 03.12.21
if(num_fesom_groups > 1) then
call MPI_Bcast(data_array, myDim_nod2d, MPI_DOUBLE_PRECISION, 0, MPI_COMM_FESOM_SAME_RANK_IN_GROUPS, MPIerr)
end if

call exchange_nod(data_array)
end if
end if
t3=MPI_Wtime()
if (ind==1) then
time_recv(1)=t3-t1
Expand Down
Loading