main_spec.tex

\documentclass[10pt,oneside]{book}

\input{utils/packages}

\input{utils/defs}

\begin{document}

\input{content/frontmatter}



\section{The OpenSHMEM Effort}\label{subsec:openshmem_effort}
\input{content/the_openshmem_effort}

\section{Programming Model Overview}\label{subsec:programming_model}
\input{content/programming_model_overview}

\section{Memory Model}\label{subsec:memory_model}
\input{content/memory_model}

\section{Execution Model}\label{subsec:execution_model}
\input{content/execution_model}

\section{Language Bindings and Conformance}\label{subsec:bindings}
\input{content/language_bindings_and_conformance}

\section{Library Constants}\label{subsec:library_constants}
\input{content/library_constants}

\section{Library Handles}\label{subsec:library_handles}
\input{content/library_handles}

\section{Environment Variables }\label{subsec:environment_variables}
\input{content/environment_variables}

\clearpage



\section{OpenSHMEM Library \acs{API}}\label{sec:openshmem_library_api}

\subsection{Library Setup, Exit, and Query Routines}
The library setup and query interfaces that initialize and monitor the parallel
environment of the \acp{PE}.

\subsubsection{\textbf{SHMEM\_INIT}}\label{subsec:shmem_init}
\input{content/shmem_init}

\subsubsection{\textbf{SHMEM\_MY\_PE}}\label{subsec:shmem_my_pe}
\input{content/shmem_my_pe}

\subsubsection{\textbf{SHMEM\_N\_PES}}\label{subsec:shmem_n_pes}
\input{content/shmem_n_pes}

\subsubsection{\textbf{SHMEM\_FINALIZE}}\label{subsec:shmem_finalize}
\input{content/shmem_finalize}

\subsubsection{\textbf{SHMEM\_GLOBAL\_EXIT}}\label{subsec:shmem_global_exit}
\input{content/shmem_global_exit}

\subsubsection{\textbf{SHMEM\_PE\_ACCESSIBLE}}\label{subsec:shmem_pe_accessible}
\input{content/shmem_pe_accessible}

\subsubsection{\textbf{SHMEM\_ADDR\_ACCESSIBLE}}\label{subsec:shmem_addr_accessible}
\input{content/shmem_addr_accessible}

\subsubsection{\textbf{SHMEM\_PTR}}\label{subsec:shmem_ptr}
\input{content/shmem_ptr}

\subsubsection{\textbf{SHMEM\_TEAM\_PTR}}\label{subsec:shmem_team_ptr}
\input{content/shmem_team_ptr}

\subsubsection{\textbf{SHMEM\_INFO\_GET\_VERSION}}\label{subsec:shmem_info_get_version}
\input{content/shmem_info_get_version}

\subsubsection{\textbf{SHMEM\_INFO\_GET\_NAME}}\label{subsec:shmem_info_get_name}
\input{content/shmem_info_get_name}

\subsubsection{\textbf{START\_PES}}\label{subsec:start_pes}
\input{content/start_pes}

\subsection{Thread Support}
\label{subsec:thread_support}
\input{content/threads_intro.tex}

\subsubsection{\textbf{SHMEM\_INIT\_THREAD}}
\label{subsec:shmem_init_thread}
\input{content/shmem_init_thread}

\subsubsection{\textbf{SHMEM\_QUERY\_THREAD}}
\label{subsec:shmem_query_thread}
\input{content/shmem_query_thread}

\subsubsection{\textbf{SHMEM\_QUERY\_INITIALIZED}}
\label{subsec:shmem_query_initialized}
\input{content/shmem_query_initialized}


\subsection{Memory Management Routines}
\label{sec:memory_management}
\input{content/memmgmt_intro.tex}

\subsubsection{\textbf{SHMEM\_MALLOC}}\label{subsec:shmem_malloc}
\input{content/shmem_malloc.tex}

\subsubsection{\textbf{SHMEM\_FREE}}\label{subsec:shmem_free}
\input{content/shmem_free.tex}

\subsubsection{\textbf{SHMEM\_REALLOC}}\label{subsec:shmem_realloc}
\input{content/shmem_realloc.tex}

\subsubsection{\textbf{SHMEM\_ALIGN}}\label{subsec:shmem_align}
\input{content/shmem_align.tex}

\subsubsection{\textbf{SHMEM\_MALLOC\_WITH\_HINTS}}\label{subsec:shmmallochint}
\input{content/shmem_malloc_hints.tex}

\subsubsection{\textbf{SHMEM\_CALLOC}}\label{subsec:shmem_calloc}
\input{content/shmem_calloc.tex}



\subsection{Team Management Routines}\label{subsec:team}
\input{content/teams_intro.tex}

\subsubsection{\textbf{SHMEM\_TEAM\_MY\_PE}}\label{subsec:shmem_team_my_pe}
\input{content/shmem_team_my_pe.tex}

\subsubsection{\textbf{SHMEM\_TEAM\_N\_PES}}\label{subsec:shmem_team_n_pes}
\input{content/shmem_team_n_pes.tex}

\subsubsection{\textbf{SHMEM\_TEAM\_CONFIG\_T}}
\label{subsec:shmem_team_config_t}
\input{content/shmem_team_config_t.tex}

\subsubsection{\textbf{SHMEM\_TEAM\_GET\_CONFIG}}\label{subsec:shmem_team_get_config}
\input{content/shmem_team_get_config.tex}

\subsubsection{\textbf{SHMEM\_TEAM\_TRANSLATE\_PE}}\label{subsec:shmem_team_translate_pe}
\input{content/shmem_team_translate_pe.tex}

\subsubsection{\textbf{SHMEM\_TEAM\_SPLIT\_STRIDED}}\label{subsec:shmem_team_split_strided}
\input{content/shmem_team_split_strided.tex}

\subsubsection{\textbf{SHMEM\_TEAM\_SPLIT\_2D}}\label{subsec:shmem_team_split_2d}
\input{content/shmem_team_split_2d.tex}

\subsubsection{\textbf{SHMEM\_TEAM\_DESTROY}}\label{subsec:shmem_team_destroy}
\input{content/shmem_team_destroy.tex}



\subsection{Communication Management Routines}\label{sec:ctx}
\input{content/context_intro.tex}

\subsubsection{\textbf{SHMEM\_CTX\_CREATE}}
\label{subsec:shmem_ctx_create}
\input{content/shmem_ctx_create.tex}

\subsubsection{\textbf{SHMEM\_TEAM\_CREATE\_CTX}}
\label{subsec:shmem_team_create_ctx}
\input{content/shmem_team_create_ctx.tex}

\subsubsection{\textbf{SHMEM\_CTX\_DESTROY}}
\label{subsec:shmem_ctx_destroy}
\input{content/shmem_ctx_destroy.tex}

\subsubsection{\textbf{SHMEM\_CTX\_GET\_TEAM}}
\label{subsec:shmem_ctx_get_team}
\input{content/shmem_ctx_get_team.tex}

\subsection{Remote Memory Access Routines}\label{sec:rma}
\input{content/rma_intro.tex}

\subsubsection{Blocking Remote Memory Access Routines}\label{subsec:rma}
\subsubsubsection{\textbf{SHMEM\_PUT}}\label{subsec:shmem_put}
\input{content/shmem_put.tex}

\subsubsubsection{\textbf{SHMEM\_P}}\label{subsec:shmem_p}
\input{content/shmem_p.tex}

\subsubsubsection{\textbf{SHMEM\_IPUT}}\label{subsec:shmem_iput}
\input{content/shmem_iput.tex}

\subsubsubsection{\textbf{SHMEM\_IBPUT}}\label{subsec:shmem_ibput}
\input{content/shmem_ibput.tex}

\subsubsubsection{\textbf{SHMEM\_GET}}\label{subsec:shmem_get}
\input{content/shmem_get.tex}

\subsubsubsection{\textbf{SHMEM\_G}}\label{subsec:shmem_g}
\input{content/shmem_g.tex}

\subsubsubsection{\textbf{SHMEM\_IGET}}\label{subsec:shmem_iget}
\input{content/shmem_iget.tex}

\subsubsubsection{\textbf{SHMEM\_IBGET}}\label{subsec:shmem_ibget}
\input{content/shmem_ibget.tex}

\subsubsection{Nonblocking Remote Memory Access Routines}\label{subsec:rma_nbi}

\subsubsubsection{\textbf{SHMEM\_PUT\_NBI}}\label{subsec:shmem_put_nbi}
\input{content/shmem_put_nbi.tex}

\subsubsubsection{\textbf{SHMEM\_GET\_NBI}}\label{subsec:shmem_get_nbi}
\input{content/shmem_get_nbi.tex}



\subsection{Atomic Memory Operations}\label{sec:amo}
\input{content/atomics_intro}

\subsubsection{Blocking Atomic Memory Operations}\label{subsec:amo}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH}}
\label{subsec:shmem_atomic_fetch}
\input{content/shmem_atomic_fetch.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_SET}}
\label{subsec:shmem_atomic_set}
\input{content/shmem_atomic_set.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_COMPARE\_SWAP}}
\label{subsec:shmem_atomic_compare_swap}
\input{content/shmem_atomic_compare_swap.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_SWAP}}
\label{subsec:shmem_atomic_swap}
\input{content/shmem_atomic_swap.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH\_INC}}
\label{subsec:shmem_atomic_fetch_inc}
\input{content/shmem_atomic_fetch_inc.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_INC}}
\label{subsec:shmem_atomic_inc}
\input{content/shmem_atomic_inc.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH\_ADD}}
\label{subsec:shmem_atomic_fetch_add}
\input{content/shmem_atomic_fetch_add.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_ADD}}
\label{subsec:shmem_atomic_add}
\input{content/shmem_atomic_add.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH\_AND}}
\label{subsec:shmem_atomic_fetch_and}
\input{content/shmem_atomic_fetch_and.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_AND}}
\label{subsec:shmem_atomic_and}
\input{content/shmem_atomic_and.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH\_OR}}
\label{subsec:shmem_atomic_fetch_or}
\input{content/shmem_atomic_fetch_or.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_OR}}
\label{subsec:shmem_atomic_or}
\input{content/shmem_atomic_or.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH\_XOR}}
\label{subsec:shmem_atomic_fetch_xor}
\input{content/shmem_atomic_fetch_xor.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_XOR}}
\label{subsec:shmem_atomic_xor}
\input{content/shmem_atomic_xor.tex}

\subsubsection{Nonblocking Atomic Memory Operations}\label{subsec:amo_nbi}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH\_NBI}}
\label{subsec:shmem_atomic_fetch_nbi}
\input{content/shmem_atomic_fetch_nbi.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_COMPARE\_SWAP\_NBI}}
\label{subsec:shmem_atomic_compare_swap_nbi}
\input{content/shmem_atomic_compare_swap_nbi.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_SWAP\_NBI}}
\label{subsec:shmem_atomic_swap_nbi}
\input{content/shmem_atomic_swap_nbi.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH\_INC\_NBI}}
\label{subsec:shmem_atomic_fetch_inc_nbi}
\input{content/shmem_atomic_fetch_inc_nbi.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH\_ADD\_NBI}}
\label{subsec:shmem_atomic_fetch_add_nbi}
\input{content/shmem_atomic_fetch_add_nbi.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH\_AND\_NBI}}
\label{subsec:shmem_atomic_fetch_and_nbi}
\input{content/shmem_atomic_fetch_and_nbi.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH\_OR\_NBI}}
\label{subsec:shmem_atomic_fetch_or_nbi}
\input{content/shmem_atomic_fetch_or_nbi.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH\_XOR\_NBI}}
\label{subsec:shmem_atomic_fetch_xor_nbi}
\input{content/shmem_atomic_fetch_xor_nbi.tex}



\subsection{Signaling Operations}\label{sec:shmem_signal}
This section specifies the OpenSHMEM support for \OPR{put-with-signal},
nonblocking \OPR{put-with-signal}, and \OPR{signal-\{add, fetch, set\}} routines. The
put-with-signal routines provide a method for copying data from a contiguous
local data object to a data object on a specified \ac{PE} and subsequently
updating a remote flag to signal completion.
The signal-add and signal-set routines provide methods for updating
the signal object without the associated data transfer of a
put-with-signal operation.
The signal-fetch routine provides support for reading a local signal value.

\openshmem \OPR{put-with-signal} and \OPR{signal-\{add, set\}}
routines specified in this section have two
variants. In one of the variants, the context handle, \VAR{ctx}, is explicitly
passed as an argument. In this variant, the operation is performed on the
specified context. If the context handle \VAR{ctx} does not correspond to a
valid context, the behavior is undefined. In the other variant, the context
handle is not explicitly passed and thus, the operations are performed on the
default context.

\subsubsection{Atomicity Guarantees for Signaling Operations}
\label{subsec:signal_atomicity}
All signaling operations put-with-signal, nonblocking put-with-signal, and
signal-\{add, fetch, set\} are performed on a signal data object, a remotely accessible
symmetric object of type \VAR{uint64\_t}. A signal operator in the
put-with-signal routine is an \openshmem library constant that determines the
type of update to be performed as a signal on the signal data object.

All signaling operations on the signal data object complete as if performed
atomically with respect to the following:
\begin{itemize}
    \item other blocking or nonblocking variant of the put-with-signal routine
    that updates the signal data object using the same signal update operator;
    \item signal-add routine when the put-with-signal routine uses the
      \LibConstRef{SHMEM\_SIGNAL\_ADD} signal operator;
    \item signal-set routine when the put-with-signal routine uses the
      \LibConstRef{SHMEM\_SIGNAL\_SET} signal operator;
    \item signal-fetch routine that fetches the signal data object; and
    \item any point-to-point synchronization routine that accesses the signal
    data object.
\end{itemize}

\subsubsection{Available Signal Operators}
\label{subsec:signal_operator}

With the atomicity guarantees as described in
Section~\ref{subsec:signal_atomicity}, the following options can be used as a
signal operator.

    \apitablerow{\LibConstRef{SHMEM\_SIGNAL\_SET}}{An update to signal data
    object is an atomic set operation. It writes an unsigned 64-bit value as a
    signal into the signal data object on a remote \VAR{PE} as an atomic
    operation.}

    \apitablerow{\LibConstRef{SHMEM\_SIGNAL\_ADD}}{An update to signal data
    object is an atomic add operation. It adds an unsigned 64-bit value as a
    signal into the signal data object on a remote \VAR{PE} as an atomic
    operation.}


\subsubsection{\textbf{SHMEM\_PUT\_SIGNAL}}\label{subsec:shmem_put_signal}
\input{content/shmem_put_signal.tex}

\subsubsection{\textbf{SHMEM\_PUT\_SIGNAL\_NBI}}\label{subsec:shmem_put_signal_nbi}
\input{content/shmem_put_signal_nbi.tex}

\subsubsection{\textbf{SHMEM\_SIGNAL\_ADD}}\label{subsec:shmem_signal_add}
\input{content/shmem_signal_add.tex}

\subsubsection{\textbf{SHMEM\_SIGNAL\_FETCH}}\label{subsec:shmem_signal_fetch}
\input{content/shmem_signal_fetch.tex}

\subsubsection{\textbf{SHMEM\_SIGNAL\_SET}}\label{subsec:shmem_signal_set}
\input{content/shmem_signal_set.tex}


\subsection{Session Routines}\label{subsec:sessions}
\input{content/sessions_intro.tex}

\subsubsection{\textbf{SHMEM\_CTX\_SESSION\_CONFIG\_T}}\label{subsec:shmem_ctx_session_config_t}
\input{content/shmem_ctx_session_config_t.tex}

\subsubsection{\textbf{SHMEM\_CTX\_SESSION\_START}}\label{subsec:shmem_ctx_session_start}
\input{content/shmem_ctx_session_start.tex}

\subsubsection{\textbf{SHMEM\_CTX\_SESSION\_STOP}}\label{subsec:shmem_ctx_session_stop}
\input{content/shmem_ctx_session_stop.tex}


\subsection{Collective Routines}\label{subsec:coll}
\input{content/collective_intro.tex}

\subsubsection{\textbf{SHMEM\_BARRIER\_ALL}}\label{subsec:shmem_barrier_all}
\input{content/shmem_barrier_all.tex}

\subsubsection{\textbf{SHMEM\_BARRIER}}\label{subsec:shmem_barrier}
\input{content/shmem_barrier.tex}

\subsubsection{\textbf{SHMEM\_SYNC}}\label{subsec:shmem_sync}
\input{content/shmem_sync.tex}

\subsubsection{\textbf{SHMEM\_SYNC\_ALL}}\label{subsec:shmem_sync_all}
\input{content/shmem_sync_all.tex}

\subsubsection{\textbf{SHMEM\_ALLTOALL}}\label{subsec:shmem_alltoall}
\input{content/shmem_alltoall.tex}

\subsubsection{\textbf{SHMEM\_ALLTOALLS}}\label{subsec:shmem_alltoalls}
\input{content/shmem_alltoalls.tex}

\subsubsection{\textbf{SHMEM\_BROADCAST}}\label{subsec:shmem_broadcast}
\input{content/shmem_broadcast.tex}

\subsubsection{\textbf{SHMEM\_COLLECT, SHMEM\_FCOLLECT}}\label{subsec:shmem_collect}
\input{content/shmem_collect.tex}

\subsubsection{\textbf{SHMEM\_REDUCTIONS}}\label{subsec:shmem_reductions}
\input{content/shmem_reductions.tex}

\subsubsection{\textbf{SHMEM\_SCAN}}\label{subsec:shmem_scan}
\input{content/shmem_scan.tex}




\subsection{Point-To-Point Synchronization Routines}\label{subsec:p2p_intro}
\input{content/p2p_sync_intro.tex}

\subsubsection{\textbf{SHMEM\_WAIT\_UNTIL}}\label{subsec:shmem_wait_until}
\input{content/shmem_wait_until.tex}

\subsubsection{\textbf{SHMEM\_WAIT\_UNTIL\_ALL}}\label{subsec:shmem_wait_until_all}
\input{content/shmem_wait_until_all.tex}

\subsubsection{\textbf{SHMEM\_WAIT\_UNTIL\_ANY}}\label{subsec:shmem_wait_until_any}
\input{content/shmem_wait_until_any.tex}

\subsubsection{\textbf{SHMEM\_WAIT\_UNTIL\_SOME}}\label{subsec:shmem_wait_until_some}
\input{content/shmem_wait_until_some.tex}

\subsubsection{\textbf{SHMEM\_WAIT\_UNTIL\_ALL\_VECTOR}}\label{subsec:shmem_wait_until_all_vector}
\input{content/shmem_wait_until_all_vector.tex}

\subsubsection{\textbf{SHMEM\_WAIT\_UNTIL\_ANY\_VECTOR}}\label{subsec:shmem_wait_until_any_vector}
\input{content/shmem_wait_until_any_vector.tex}

\subsubsection{\textbf{SHMEM\_WAIT\_UNTIL\_SOME\_VECTOR}}\label{subsec:shmem_wait_until_some_vector}
\input{content/shmem_wait_until_some_vector.tex}

\subsubsection{\textbf{SHMEM\_TEST}}\label{subsec:shmem_test}
\input{content/shmem_test.tex}

\subsubsection{\textbf{SHMEM\_TEST\_ALL}}\label{subsec:shmem_test_all}
\input{content/shmem_test_all.tex}

\subsubsection{\textbf{SHMEM\_TEST\_ANY}}\label{subsec:shmem_test_any}
\input{content/shmem_test_any.tex}

\subsubsection{\textbf{SHMEM\_TEST\_SOME}}\label{subsec:shmem_test_some}
\input{content/shmem_test_some.tex}

\subsubsection{\textbf{SHMEM\_TEST\_ALL\_VECTOR}}\label{subsec:shmem_test_all_vector}
\input{content/shmem_test_all_vector.tex}

\subsubsection{\textbf{SHMEM\_TEST\_ANY\_VECTOR}}\label{subsec:shmem_test_any_vector}
\input{content/shmem_test_any_vector.tex}

\subsubsection{\textbf{SHMEM\_TEST\_SOME\_VECTOR}}\label{subsec:shmem_test_some_vector}
\input{content/shmem_test_some_vector.tex}

\subsubsection{\textbf{SHMEM\_SIGNAL\_WAIT\_UNTIL}}\label{subsec:shmem_signal_wait_until}
\input{content/shmem_signal_wait_until.tex}



\subsection{Memory Ordering Routines}\label{subsec:memory_order}
The following section discusses \openshmem \acp{API} that provide mechanisms to
ensure ordering and/or delivery of completion on memory store, blocking, and
nonblocking \openshmem routines. Table~\ref{mem-order} lists the operations
affected by \openshmem memory ordering routines.

\begin{longtable}{|l|c|c|}
 \hline
  Operations & Fence & Quiet \\ \hline
  Memory Store \footnote{Ordering and/or delivery of memory store operations are
  ensured only on contexts created with certain options. For details, refer the
  description of context options in Section~\ref{subsec:shmem_ctx_create}.}
                                     & X & X \\
  Blocking \OPR{Put}                 & X & X \\
  Blocking \OPR{Get}                 &   &   \\
  Blocking \OPR{\acs{AMO}}           & X & X \\
  Blocking \OPR{put-with-signal}     & X & X \\
  Nonblocking \OPR{Put}              & X & X \\
  Nonblocking \OPR{Get}              &   & X \\
  Nonblocking \OPR{\acs{AMO}}        & X\footnote{\openshmem fence routines does
  not guarantee order of delivery of values fetched by nonblocking \ac{AMO}
  routines.}
                                         & X \\
  Nonblocking \OPR{put-with-signal}  & X & X \\ \hline
\TableCaptionRef{List of operations affected by \openshmem Memory Ordering
routines}
\label{mem-order}
\end{longtable}



\subsubsection{\textbf{SHMEM\_FENCE}}\label{subsec:shmem_fence}
\input{content/shmem_fence.tex}

\subsubsection{\textbf{SHMEM\_QUIET}}\label{subsec:shmem_quiet}
\input{content/shmem_quiet.tex}

\subsubsection{\textbf{SHMEM\_PE\_QUIET}}\label{subsec:shmem_pe_quiet}
\input{content/shmem_pe_quiet.tex}

\subsubsection{Synchronization and Communication Ordering in OpenSHMEM}
\input{content/synchronization_model.tex}






\subsection{Distributed Locking Routines}
The following section discusses \openshmem locks as a mechanism to provide
mutual exclusion. Three routines are available for distributed locking,
\textit{set, test} and \textit{clear}.

\subsubsection{\textbf{SHMEM\_LOCK}}\label{subsec:shmem_lock}
\input{content/shmem_lock.tex}





\section{OpenSHMEM Profiling Interface}\label{sec:openshmem_profiling_interface}
\input{content/profiling_interface.tex}

\clearpage
\input{content/backmatter}

\chapter*{Glossary}
\addcontentsline{toc}{chapter}{Glossary}
\input{content/glossary.tex}

\clearpage
\phantomsection
\addcontentsline{toc}{chapter}{Index}
\printindex

\end{document}