tem_comm_module Module

  1. tem_comm_module.f90
  2. tem_comm_module

This module provides the data structure for the communication during the simulation.

Several exchange methods are implemented. CoCo is heavily used here to allow for a concise definition of exchanges for various data types. The basic idea is to initialize the buffers, use them in exchanges and finalize them when not needed anymore. In the definition of the buffers, an array of positions in the original linearized data array is used to describe the origin or target positions for communicated data.


Uses

  • module~~tem_comm_module~~UsesGraph module~tem_comm_module tem_comm_module module~tem_dyn_array_module tem_dyn_array_module module~tem_comm_module->module~tem_dyn_array_module module~tem_aux_module tem_aux_module module~tem_comm_module->module~tem_aux_module iso_c_binding iso_c_binding module~tem_comm_module->iso_c_binding module~env_module env_module module~tem_comm_module->module~env_module module~tem_logging_module tem_logging_module module~tem_comm_module->module~tem_logging_module module~flu_binding flu_binding module~tem_comm_module->module~flu_binding hvs_sizeof_module hvs_sizeof_module module~tem_comm_module->hvs_sizeof_module tem_sparse_comm_module tem_sparse_comm_module module~tem_comm_module->tem_sparse_comm_module module~tem_grow_array_module tem_grow_array_module module~tem_comm_module->module~tem_grow_array_module module~aotus_module aotus_module module~tem_comm_module->module~aotus_module mem_for_mpi_module mem_for_mpi_module module~tem_comm_module->mem_for_mpi_module mpi mpi module~tem_comm_module->mpi module~tem_dyn_array_module->module~env_module module~tem_aux_module->module~env_module module~tem_aux_module->module~tem_logging_module module~tem_aux_module->module~flu_binding module~tem_aux_module->module~aotus_module module~tem_aux_module->mpi module~tem_revision_module tem_revision_module module~tem_aux_module->module~tem_revision_module module~tem_comm_env_module tem_comm_env_module module~tem_aux_module->module~tem_comm_env_module module~aot_table_module aot_table_module module~tem_aux_module->module~aot_table_module module~tem_tools_module tem_tools_module module~tem_aux_module->module~tem_tools_module module~tem_lua_requires_module tem_lua_requires_module module~tem_aux_module->module~tem_lua_requires_module module~env_module->module~flu_binding module~env_module->module~aotus_module module~env_module->mpi iso_fortran_env iso_fortran_env module~env_module->iso_fortran_env module~tem_logging_module->module~env_module module~tem_logging_module->module~aotus_module module~tem_logging_module->module~aot_table_module module~tem_grow_array_module->module~env_module module~tem_comm_env_module->mpi module~tem_tools_module->module~env_module module~tem_lua_requires_module->iso_c_binding module~tem_lua_requires_module->module~env_module module~tem_lua_requires_module->module~flu_binding module~tem_lua_requires_module->module~aotus_module module~tem_lua_requires_module->module~aot_table_module

Used by

  • module~~tem_comm_module~~UsedByGraph module~tem_comm_module tem_comm_module module~tem_stencil_module tem_stencil_module module~tem_stencil_module->module~tem_comm_module module~tem_general_module tem_general_module module~tem_general_module->module~tem_comm_module module~tem_coupling_module tem_coupling_module module~tem_coupling_module->module~tem_comm_module module~tem_face_module tem_face_module module~tem_face_module->module~tem_comm_module module~tem_facedata_module tem_faceData_module module~tem_facedata_module->module~tem_comm_module program~tem_face_test~2 tem_face_test program~tem_face_test~2->module~tem_comm_module module~tem_construction_module tem_construction_module module~tem_construction_module->module~tem_comm_module

Contents

Abstract Interfaces

Derived Types

Subroutines


Abstract Interfaces

abstract interface

  • private subroutine tem_exchange_long(send, recv, state, message_flag, send_state, comm)

    Arguments

    TypeIntentOptionalAttributesName
    type(tem_communication_type), intent(inout) :: send
    type(tem_communication_type), intent(inout) :: recv
    integer(kind=long_k), intent(inout) :: state(*)
    integer, intent(in) :: message_flag
    integer(kind=long_k), intent(in), optional :: send_state(*)
    integer, intent(in) :: comm

    mpi communicator

abstract interface

  • private subroutine tem_commbuf_long_init(me, pos, nvals)

    Arguments

    TypeIntentOptionalAttributesName
    type(tem_longbuffer_type), intent(inout) :: me
    integer, intent(in) :: pos(nvals)
    integer, intent(in) :: nvals

abstract interface

  • private subroutine tem_commbuf_long_fin(me)

    Arguments

    TypeIntentOptionalAttributesName
    type(tem_longbuffer_type), intent(inout) :: me

abstract interface

  • private subroutine tem_exchange_int(send, recv, state, message_flag, send_state, comm)

    Arguments

    TypeIntentOptionalAttributesName
    type(tem_communication_type), intent(inout) :: send
    type(tem_communication_type), intent(inout) :: recv
    integer, intent(inout) :: state(*)
    integer, intent(in) :: message_flag
    integer, intent(in), optional :: send_state(*)
    integer, intent(in) :: comm

    mpi communicator

abstract interface

  • private subroutine tem_commbuf_int_init(me, pos, nvals)

    Arguments

    TypeIntentOptionalAttributesName
    type(tem_intbuffer_type), intent(inout) :: me
    integer, intent(in) :: pos(nvals)
    integer, intent(in) :: nvals

abstract interface

  • private subroutine tem_commbuf_int_fin(me)

    Arguments

    TypeIntentOptionalAttributesName
    type(tem_intbuffer_type), intent(inout) :: me

abstract interface

  • private subroutine tem_exchange_real(send, recv, state, message_flag, send_state, comm)

    Arguments

    TypeIntentOptionalAttributesName
    type(tem_communication_type), intent(inout) :: send
    type(tem_communication_type), intent(inout) :: recv
    real(kind=rk), intent(inout) :: state(*)
    integer, intent(in) :: message_flag
    real(kind=rk), intent(in), optional :: send_state(*)
    integer, intent(in) :: comm

    mpi communicator

abstract interface

  • private subroutine tem_commbuf_real_init(me, pos, nvals)

    Arguments

    TypeIntentOptionalAttributesName
    type(tem_realbuffer_type), intent(inout) :: me
    integer, intent(in) :: pos(nvals)
    integer, intent(in) :: nvals

abstract interface

  • private subroutine tem_commbuf_real_fin(me)

    Arguments

    TypeIntentOptionalAttributesName
    type(tem_realbuffer_type), intent(inout) :: me

Derived Types

type, public :: tem_longbuffer_type

process-wise buffer for data of type integer(kind=long_k)

Read more…

Components

TypeVisibilityAttributesNameInitial
integer(kind=long_k), private, pointer:: val(:)=> null()

explicit buffer for data to be transferred
type(c_ptr), private :: mem_mpi

explicit buffer in memory allocated by mpi
integer, private, allocatable:: pos(:)

position in the input vector from where to read the entries in val_long

Read more…
integer, private :: nvals

number of values to exchange

Read more…
integer, private :: memindexed

handle for the mpi-datatype to describe the memory access, without explicit copying in the application.

type, public :: tem_intbuffer_type

process-wise buffer for data of type integer

Read more…

Components

TypeVisibilityAttributesNameInitial
integer, private, pointer:: val(:)=> null()

explicit buffer for data to be transferred
type(c_ptr), private :: mem_mpi

explicit buffer in memory allocated by mpi
integer, private, allocatable:: pos(:)

position in the input vector from where to read the entries in val_int

Read more…
integer, private :: nvals

number of values to exchange

Read more…
integer, private :: memindexed

handle for the mpi-datatype to describe the memory access, without explicit copying in the application.

type, public :: tem_realbuffer_type

process-wise buffer for data of type real(kind=rk)

Read more…

Components

TypeVisibilityAttributesNameInitial
real(kind=rk), private, pointer:: val(:)=> null()

explicit buffer for data to be transferred
type(c_ptr), private :: mem_mpi

explicit buffer in memory allocated by mpi
integer, private, allocatable:: pos(:)

position in the input vector from where to read the entries in val_real

Read more…
integer, private :: nvals

number of values to exchange

Read more…
integer, private :: memindexed

handle for the mpi-datatype to describe the memory access, without explicit copying in the application.

type, public :: tem_communication_type

Description of communication data

Components

TypeVisibilityAttributesNameInitial
integer, private :: nProcs =0
integer, private, allocatable:: proc(:)

partition MPI rank
integer, private, allocatable:: nElemsProc(:)

How many data elements need to be exchanged with proc (per process).
integer, private, allocatable:: rqHandle(:)

Request handle array
type(grw_intarray_type), private, allocatable:: elemPos(:)

Data element positions in the actual arrays, used to built the pos information in the actual buffers (per process).
type(tem_longbuffer_type), private, allocatable:: buf_long(:)

declare communication buffers for each variable type
type(tem_intbuffer_type), private, allocatable:: buf_int(:)
type(tem_realbuffer_type), private, allocatable:: buf_real(:)

type, public :: tem_commPattern_type

General description of the communication pattern to use.

Read more…

Components

TypeVisibilityAttributesNameInitial
character(len=40), private :: style
procedure(tem_exchange_long), private, nopass, pointer:: exchange_long
procedure(tem_commbuf_long_init), private, nopass, pointer:: initbuf_long
procedure(tem_commbuf_long_fin), private, nopass, pointer:: finbuf_long
procedure(tem_exchange_int), private, nopass, pointer:: exchange_int
procedure(tem_commbuf_int_init), private, nopass, pointer:: initbuf_int
procedure(tem_commbuf_int_fin), private, nopass, pointer:: finbuf_int
procedure(tem_exchange_real), private, nopass, pointer:: exchange_real
procedure(tem_commbuf_real_init), private, nopass, pointer:: initbuf_real
procedure(tem_commbuf_real_fin), private, nopass, pointer:: finbuf_real

Subroutines

public subroutine tem_load_commPattern(me, conf, style)

This subroutine loads the communication pattern from a Lua script and sets the exchange routine to be used accordingly.

Read more…

Arguments

TypeIntentOptionalAttributesName
type(tem_commPattern_type), intent(out) :: me

commpattern to set
type(flu_State), optional :: conf

handle to the Lua script
character(len=*), intent(in), optional :: style

optional communication style

public subroutine tem_comm_dumpType(me, nUnit)

Write communication type data to nUnit (debugging routine)

Arguments

TypeIntentOptionalAttributesName
type(tem_communication_type), intent(in) :: me
integer, intent(in) :: nUnit

public subroutine tem_comm_init(me, nProcs)

Allocate tem_communication_type and its variables

Arguments

TypeIntentOptionalAttributesName
type(tem_communication_type), intent(inout) :: me
integer, intent(in) :: nProcs

public subroutine tem_comm_count(me, comm_size, nHalos)

Allocate tem_communication_type and its variables

Arguments

TypeIntentOptionalAttributesName
type(tem_communication_type), intent(inout) :: me
integer, intent(in) :: comm_size

communicator size
integer, intent(in) :: nHalos(comm_size)

number of halos for each other processes

public subroutine tem_comm_createBuffer(commBuffer, nScalars, nElems, elemRanks)

Routine to build communication buffer using elemRanks. This routine can be used only if all elements need to be communicated but they need process-wise seperation. Uses nScalars to get position in the value array to communicate. For send buffer: elemRanks contains target ranks to send data to For recv buffer: elemRanks contains source ranks to recv data from

Arguments

TypeIntentOptionalAttributesName
type(tem_communication_type), intent(out) :: commBuffer

send or recv communication buffer to be created
integer, intent(in) :: nScalars

Number of scalars per element
integer, intent(in) :: nElems

Total number of elements or points to communicate
integer, intent(in) :: elemRanks(nElems)

Target or source rank for each element or point

public subroutine tem_comm_alltoall_int(targets, send_buffer, sources, recv_buffer, comm, tag)

All to all exchange of a single integer.

Read more…

Arguments

TypeIntentOptionalAttributesName
integer, intent(in) :: targets(:)

List of target ranks to send an integer to.
integer, intent(in) :: send_buffer(:)

Data to send to the respective target ranks. This array has to have the same ordering as targets.
integer, intent(out), allocatable:: sources(:)

List of ranks we received data from (source ranks). The array will be allocated with a size according to the number of processes that send a request to this process.
integer, intent(out), allocatable:: recv_buffer(:)

Received data from the sources. The array has the same size and ordering as the sources array.
integer, intent(in) :: comm

MPI Communicator to use for this data exchange.
integer, intent(in), optional :: tag

Tag to use in the communications. Defaults to 22.

public subroutine tem_comm_destroy(me, commPattern)

Arguments

TypeIntentOptionalAttributesName
type(tem_communication_type), intent(inout) :: me

communication type to be destroyed
type(tem_commPattern_type), intent(in) :: commPattern

Communication pattern

private subroutine tem_commbuf_long_fillpos(me, pos, nvals)

fill the positions that describe how the data in the state vector relates to the entries in the buffer.

Arguments

TypeIntentOptionalAttributesName
type(tem_longbuffer_type), intent(inout) :: me
integer, intent(in) :: pos(nvals)
integer, intent(in) :: nvals

private subroutine tem_commbuf_long_finpos(me)

free the communication buffer allocated by the fillpos routine again.

Arguments

TypeIntentOptionalAttributesName
type(tem_longbuffer_type), intent(inout) :: me

private subroutine tem_commbuf_long_fillmpimem(me, pos, nvals)

fill the positions that describe how the data in the state vector relates to the entries in the buffer use memory that is allocated by mpi for the buffer.

Arguments

TypeIntentOptionalAttributesName
type(tem_longbuffer_type), intent(inout) :: me
integer, intent(in) :: pos(nvals)
integer, intent(in) :: nvals

private subroutine tem_commbuf_long_finmpimem(me)

free the communication buffer allocated by the fillmpimem routine again.

Arguments

TypeIntentOptionalAttributesName
type(tem_longbuffer_type), intent(inout) :: me

private subroutine tem_commbuf_long_fillindexed(me, pos, nvals)

fill the indexed mpi datatype, which describes how the data in the state vector relates to the entries in the buffer.

Arguments

TypeIntentOptionalAttributesName
type(tem_longbuffer_type), intent(inout) :: me
integer, intent(in) :: pos(nvals)
integer, intent(in) :: nvals

private subroutine tem_commbuf_long_fintyped(me)

free the communication buffer allocated by the fillindexed routine again.

Arguments

TypeIntentOptionalAttributesName
type(tem_longbuffer_type), intent(inout) :: me

private subroutine tem_commbuf_long_gatherindexed(me, pos, nvals)

gather the indexed mpi datatype, which describes how the data in the state vector relates to the entries in the buffer. in contrast to the simple indexed type above, we try to minimize the number of blocks here, and gather contiguous blocks of memory together.

Arguments

TypeIntentOptionalAttributesName
type(tem_longbuffer_type), intent(inout) :: me
integer, intent(in) :: pos(nvals)
integer, intent(in) :: nvals

private subroutine comm_isend_irecv_long(send, recv, state, message_flag, send_state, comm)

exchange the communication buffers with a non-blocking mpi communication using preposted irecv and isend with a waitall

Arguments

TypeIntentOptionalAttributesName
type(tem_communication_type), intent(inout) :: send
type(tem_communication_type), intent(inout) :: recv
integer(kind=long_k), intent(inout) :: state(*)
integer, intent(in) :: message_flag
integer(kind=long_k), intent(in), optional :: send_state(*)
integer, intent(in) :: comm

mpi communicator

private subroutine comm_isend_irecv_overlap_long(send, recv, state, message_flag, send_state, comm)

fill send buffers and start sending

Arguments

TypeIntentOptionalAttributesName
type(tem_communication_type), intent(inout) :: send
type(tem_communication_type), intent(inout) :: recv
integer(kind=long_k), intent(inout) :: state(*)
integer, intent(in) :: message_flag
integer(kind=long_k), intent(in), optional :: send_state(*)
integer, intent(in) :: comm

mpi communicator

private subroutine comm_typed_isend_irecv_long(send, recv, state, message_flag, send_state, comm)

exchange the communication mes with a non-blocking mpi communication using preposted irecv and isend with a waitall

Read more…

Arguments

TypeIntentOptionalAttributesName
type(tem_communication_type), intent(inout) :: send
type(tem_communication_type), intent(inout) :: recv
integer(kind=long_k), intent(inout) :: state(*)
integer, intent(in) :: message_flag
integer(kind=long_k), intent(in), optional :: send_state(*)
integer, intent(in) :: comm

mpi communicator

private subroutine tem_commbuf_int_fillpos(me, pos, nvals)

fill the positions that describe how the data in the state vector relates to the entries in the buffer.

Arguments

TypeIntentOptionalAttributesName
type(tem_intbuffer_type), intent(inout) :: me
integer, intent(in) :: pos(nvals)
integer, intent(in) :: nvals

private subroutine tem_commbuf_int_finpos(me)

free the communication buffer allocated by the fillpos routine again.

Arguments

TypeIntentOptionalAttributesName
type(tem_intbuffer_type), intent(inout) :: me

private subroutine tem_commbuf_int_fillmpimem(me, pos, nvals)

fill the positions that describe how the data in the state vector relates to the entries in the buffer use memory that is allocated by mpi for the buffer.

Arguments

TypeIntentOptionalAttributesName
type(tem_intbuffer_type), intent(inout) :: me
integer, intent(in) :: pos(nvals)
integer, intent(in) :: nvals

private subroutine tem_commbuf_int_finmpimem(me)

free the communication buffer allocated by the fillmpimem routine again.

Arguments

TypeIntentOptionalAttributesName
type(tem_intbuffer_type), intent(inout) :: me

private subroutine tem_commbuf_int_fillindexed(me, pos, nvals)

fill the indexed mpi datatype, which describes how the data in the state vector relates to the entries in the buffer.

Arguments

TypeIntentOptionalAttributesName
type(tem_intbuffer_type), intent(inout) :: me
integer, intent(in) :: pos(nvals)
integer, intent(in) :: nvals

private subroutine tem_commbuf_int_fintyped(me)

free the communication buffer allocated by the fillindexed routine again.

Arguments

TypeIntentOptionalAttributesName
type(tem_intbuffer_type), intent(inout) :: me

private subroutine tem_commbuf_int_gatherindexed(me, pos, nvals)

gather the indexed mpi datatype, which describes how the data in the state vector relates to the entries in the buffer. in contrast to the simple indexed type above, we try to minimize the number of blocks here, and gather contiguous blocks of memory together.

Arguments

TypeIntentOptionalAttributesName
type(tem_intbuffer_type), intent(inout) :: me
integer, intent(in) :: pos(nvals)
integer, intent(in) :: nvals

private subroutine comm_isend_irecv_int(send, recv, state, message_flag, send_state, comm)

exchange the communication buffers with a non-blocking mpi communication using preposted irecv and isend with a waitall

Arguments

TypeIntentOptionalAttributesName
type(tem_communication_type), intent(inout) :: send
type(tem_communication_type), intent(inout) :: recv
integer, intent(inout) :: state(*)
integer, intent(in) :: message_flag
integer, intent(in), optional :: send_state(*)
integer, intent(in) :: comm

mpi communicator

private subroutine comm_isend_irecv_overlap_int(send, recv, state, message_flag, send_state, comm)

fill send buffers and start sending

Arguments

TypeIntentOptionalAttributesName
type(tem_communication_type), intent(inout) :: send
type(tem_communication_type), intent(inout) :: recv
integer, intent(inout) :: state(*)
integer, intent(in) :: message_flag
integer, intent(in), optional :: send_state(*)
integer, intent(in) :: comm

mpi communicator

private subroutine comm_typed_isend_irecv_int(send, recv, state, message_flag, send_state, comm)

exchange the communication mes with a non-blocking mpi communication using preposted irecv and isend with a waitall

Read more…

Arguments

TypeIntentOptionalAttributesName
type(tem_communication_type), intent(inout) :: send
type(tem_communication_type), intent(inout) :: recv
integer, intent(inout) :: state(*)
integer, intent(in) :: message_flag
integer, intent(in), optional :: send_state(*)
integer, intent(in) :: comm

mpi communicator

private subroutine tem_commbuf_real_fillpos(me, pos, nvals)

fill the positions that describe how the data in the state vector relates to the entries in the buffer.

Arguments

TypeIntentOptionalAttributesName
type(tem_realbuffer_type), intent(inout) :: me
integer, intent(in) :: pos(nvals)
integer, intent(in) :: nvals

private subroutine tem_commbuf_real_finpos(me)

free the communication buffer allocated by the fillpos routine again.

Arguments

TypeIntentOptionalAttributesName
type(tem_realbuffer_type), intent(inout) :: me

private subroutine tem_commbuf_real_fillmpimem(me, pos, nvals)

fill the positions that describe how the data in the state vector relates to the entries in the buffer use memory that is allocated by mpi for the buffer.

Arguments

TypeIntentOptionalAttributesName
type(tem_realbuffer_type), intent(inout) :: me
integer, intent(in) :: pos(nvals)
integer, intent(in) :: nvals

private subroutine tem_commbuf_real_finmpimem(me)

free the communication buffer allocated by the fillmpimem routine again.

Arguments

TypeIntentOptionalAttributesName
type(tem_realbuffer_type), intent(inout) :: me

private subroutine tem_commbuf_real_fillindexed(me, pos, nvals)

fill the indexed mpi datatype, which describes how the data in the state vector relates to the entries in the buffer.

Arguments

TypeIntentOptionalAttributesName
type(tem_realbuffer_type), intent(inout) :: me
integer, intent(in) :: pos(nvals)
integer, intent(in) :: nvals

private subroutine tem_commbuf_real_fintyped(me)

free the communication buffer allocated by the fillindexed routine again.

Arguments

TypeIntentOptionalAttributesName
type(tem_realbuffer_type), intent(inout) :: me

private subroutine tem_commbuf_real_gatherindexed(me, pos, nvals)

gather the indexed mpi datatype, which describes how the data in the state vector relates to the entries in the buffer. in contrast to the simple indexed type above, we try to minimize the number of blocks here, and gather contiguous blocks of memory together.

Arguments

TypeIntentOptionalAttributesName
type(tem_realbuffer_type), intent(inout) :: me
integer, intent(in) :: pos(nvals)
integer, intent(in) :: nvals

private subroutine comm_isend_irecv_real(send, recv, state, message_flag, send_state, comm)

exchange the communication buffers with a non-blocking mpi communication using preposted irecv and isend with a waitall

Arguments

TypeIntentOptionalAttributesName
type(tem_communication_type), intent(inout) :: send
type(tem_communication_type), intent(inout) :: recv
real(kind=rk), intent(inout) :: state(*)
integer, intent(in) :: message_flag
real(kind=rk), intent(in), optional :: send_state(*)
integer, intent(in) :: comm

mpi communicator

private subroutine comm_isend_irecv_overlap_real(send, recv, state, message_flag, send_state, comm)

fill send buffers and start sending

Arguments

TypeIntentOptionalAttributesName
type(tem_communication_type), intent(inout) :: send
type(tem_communication_type), intent(inout) :: recv
real(kind=rk), intent(inout) :: state(*)
integer, intent(in) :: message_flag
real(kind=rk), intent(in), optional :: send_state(*)
integer, intent(in) :: comm

mpi communicator

private subroutine comm_typed_isend_irecv_real(send, recv, state, message_flag, send_state, comm)

exchange the communication mes with a non-blocking mpi communication using preposted irecv and isend with a waitall

Read more…

Arguments

TypeIntentOptionalAttributesName
type(tem_communication_type), intent(inout) :: send
type(tem_communication_type), intent(inout) :: recv
real(kind=rk), intent(inout) :: state(*)
integer, intent(in) :: message_flag
real(kind=rk), intent(in), optional :: send_state(*)
integer, intent(in) :: comm

mpi communicator