Skip to content

Commit

Permalink
Merge pull request #75 from vchoi-hdfgroup/feature/select_io_tconv
Browse files Browse the repository at this point in the history 
Feature/select io tconv
  • Loading branch information
@vchoi-hdfgroup
vchoi-hdfgroup authored Apr 24, 2023
2 parents cd6d308 + 0f4c8c5 commit c63610b
Show file tree
Hide file tree
Showing 18 changed files with 418 additions and 169 deletions.
42 changes: 39 additions & 3 deletions src/H5Dchunk.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1117,6 +1117,26 @@ H5D__chunk_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo)
if (H5D__chunk_may_use_select_io(io_info, dinfo) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't check if selection I/O is possible")

/* Calculate type conversion buffer size if necessary */
if (!(dinfo->type_info.is_xform_noop && dinfo->type_info.is_conv_noop)) {
H5SL_node_t *chunk_node; /* Current node in chunk skip list */

/* Iterate through nodes in chunk skip list */
chunk_node = H5D_CHUNK_GET_FIRST_NODE(dinfo);
while (chunk_node) {
H5D_piece_info_t *piece_info; /* Chunk information */

/* Get the actual chunk information from the skip list node */
piece_info = H5D_CHUNK_GET_NODE_INFO(dinfo, chunk_node);

/* Handle type conversion buffer */
H5D_INIT_PIECE_TCONV(io_info, dinfo, piece_info)

/* Advance to next chunk in list */
chunk_node = H5D_CHUNK_GET_NEXT_NODE(dinfo, chunk_node);
}
}

done:
if (file_space_normalized == TRUE)
if (H5S_hyper_denormalize_offset(dinfo->file_space, old_offset) < 0)
Expand Down Expand Up @@ -1568,6 +1588,10 @@ H5D__create_piece_map_single(H5D_dset_io_info_t *di, H5D_io_info_t *io_info)
/* Indicate that the chunk's memory dataspace is shared */
piece_info->mspace_shared = TRUE;

/* Initialize in-place type conversion info. Start with it disabled. */
piece_info->in_place_tconv = FALSE;
piece_info->buf_off = 0;

/* make connection to related dset info from this piece_info */
piece_info->dset_info = di;

Expand Down Expand Up @@ -1697,6 +1721,10 @@ H5D__create_piece_file_map_all(H5D_dset_io_info_t *di, H5D_io_info_t *io_info)
/* make connection to related dset info from this piece_info */
new_piece_info->dset_info = di;

/* Initialize in-place type conversion info. Start with it disabled. */
new_piece_info->in_place_tconv = FALSE;
new_piece_info->buf_off = 0;

/* Insert the new chunk into the skip list */
if (H5SL_insert(fm->dset_sel_pieces, new_piece_info, &new_piece_info->index) < 0) {
H5D__free_piece_info(new_piece_info, NULL, NULL);
Expand Down Expand Up @@ -1893,6 +1921,10 @@ H5D__create_piece_file_map_hyper(H5D_dset_io_info_t *dinfo, H5D_io_info_t *io_in
/* make connection to related dset info from this piece_info */
new_piece_info->dset_info = dinfo;

/* Initialize in-place type conversion info. Start with it disabled. */
new_piece_info->in_place_tconv = FALSE;
new_piece_info->buf_off = 0;

/* Add piece to global piece_count */
io_info->piece_count++;

Expand Down Expand Up @@ -2268,6 +2300,10 @@ H5D__piece_file_cb(void H5_ATTR_UNUSED *elem, const H5T_t H5_ATTR_UNUSED *type,
H5MM_memcpy(piece_info->scaled, scaled, sizeof(hsize_t) * fm->f_ndims);
piece_info->scaled[fm->f_ndims] = 0;

/* Initialize in-place type conversion info. Start with it disabled. */
piece_info->in_place_tconv = FALSE;
piece_info->buf_off = 0;

/* Make connection to related dset info from this piece_info */
piece_info->dset_info = dinfo;

Expand Down Expand Up @@ -2562,7 +2598,7 @@ H5D__chunk_may_use_select_io(H5D_io_info_t *io_info, const H5D_dset_io_info_t *d
/* Don't use selection I/O if there are filters on the dataset (for now) */
if (dataset->shared->dcpl_cache.pline.nused > 0) {
io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF;
io_info->no_selection_io_cause |= H5D_DATASET_FILTER;
io_info->no_selection_io_cause |= H5D_SEL_IO_DATASET_FILTER;
}
else {
hbool_t page_buf_enabled;
Expand All @@ -2573,7 +2609,7 @@ H5D__chunk_may_use_select_io(H5D_io_info_t *io_info, const H5D_dset_io_info_t *d
if (page_buf_enabled) {
/* Note that page buffer is disabled in parallel */
io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF;
io_info->no_selection_io_cause |= H5D_PAGE_BUFFER;
io_info->no_selection_io_cause |= H5D_SEL_IO_PAGE_BUFFER;
}
else {
/* Check if chunks in this dataset may be cached, if so don't use
Expand All @@ -2591,7 +2627,7 @@ H5D__chunk_may_use_select_io(H5D_io_info_t *io_info, const H5D_dset_io_info_t *d
H5_CHECK_OVERFLOW(dataset->shared->layout.u.chunk.size, uint32_t, size_t);
if ((size_t)dataset->shared->layout.u.chunk.size <= dataset->shared->cache.chunk.nbytes_max) {
io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF;
io_info->no_selection_io_cause |= H5D_CHUNK_CACHE;
io_info->no_selection_io_cause |= H5D_SEL_IO_CHUNK_CACHE;
}
#ifdef H5_HAVE_PARALLEL
} /* end else */
Expand Down
2 changes: 1 addition & 1 deletion src/H5Dcompact.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ H5D__compact_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo)

/* Disable selection I/O */
io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF;
io_info->no_selection_io_cause |= H5D_NOT_CONTIGUOUS_OR_CHUNKED_DATASET;
io_info->no_selection_io_cause |= H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET;

FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5D__compact_io_init() */
Expand Down
21 changes: 16 additions & 5 deletions src/H5Dcontig.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -665,6 +665,14 @@ H5D__contig_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo)
/* get dset file address for piece */
new_piece_info->faddr = dinfo->dset->shared->layout.storage.u.contig.addr;

/* Initialize in-place type conversion info. Start with it disabled. */
new_piece_info->in_place_tconv = FALSE;
new_piece_info->buf_off = 0;

/* Calculate type conversion buffer size and check for in-place conversion if necessary */
if (!(dinfo->type_info.is_xform_noop && dinfo->type_info.is_conv_noop))
H5D_INIT_PIECE_TCONV(io_info, dinfo, new_piece_info)

/* Save piece to dataset info struct so it is freed at the end of the
* operation */
dinfo->layout_io_info.contig_piece_info = new_piece_info;
Expand Down Expand Up @@ -760,11 +768,14 @@ H5D__contig_may_use_select_io(H5D_io_info_t *io_info, const H5D_dset_io_info_t *

/* Don't use selection I/O if it's globally disabled, if it's not a contiguous dataset, or if the sieve
* buffer exists (write) or is dirty (read) */
if (dset_info->layout_ops.readvv != H5D__contig_readvv ||
(op_type == H5D_IO_OP_READ && dataset->shared->cache.contig.sieve_dirty) ||
(op_type == H5D_IO_OP_WRITE && dataset->shared->cache.contig.sieve_buf)) {
if (dset_info->layout_ops.readvv != H5D__contig_readvv) {
io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF;
io_info->no_selection_io_cause |= H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET;
}
else if ((op_type == H5D_IO_OP_READ && dataset->shared->cache.contig.sieve_dirty) ||
(op_type == H5D_IO_OP_WRITE && dataset->shared->cache.contig.sieve_buf)) {
io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF;
io_info->no_selection_io_cause |= H5D_CONTIGUOUS_SIEVE_BUFFER;
io_info->no_selection_io_cause |= H5D_SEL_IO_CONTIGUOUS_SIEVE_BUFFER;
}
else {
hbool_t page_buf_enabled;
Expand All @@ -776,7 +787,7 @@ H5D__contig_may_use_select_io(H5D_io_info_t *io_info, const H5D_dset_io_info_t *
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't check if page buffer is enabled")
if (page_buf_enabled) {
io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF;
io_info->no_selection_io_cause |= H5D_PAGE_BUFFER;
io_info->no_selection_io_cause |= H5D_SEL_IO_PAGE_BUFFER;
}
} /* end else */

Expand Down
2 changes: 1 addition & 1 deletion src/H5Defl.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ H5D__efl_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo)

/* Disable selection I/O */
io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF;
io_info->no_selection_io_cause |= H5D_NOT_CONTIGUOUS_OR_CHUNKED_DATASET;
io_info->no_selection_io_cause |= H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET;

FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5D__efl_io_init() */
Expand Down
78 changes: 40 additions & 38 deletions src/H5Dio.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@
/********************/

/* Setup/teardown routines */
static herr_t H5D__ioinfo_init(size_t count, H5D_dset_io_info_t *dset_info, H5D_io_info_t *io_info);
static herr_t H5D__ioinfo_init(size_t count, H5D_io_op_type_t op_type, H5D_dset_io_info_t *dset_info,
H5D_io_info_t *io_info);
static herr_t H5D__dset_ioinfo_init(H5D_t *dset, H5D_dset_io_info_t *dset_info, H5D_storage_t *store);
static herr_t H5D__typeinfo_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info, hid_t mem_type_id);
#ifdef H5_HAVE_PARALLEL
Expand Down Expand Up @@ -107,9 +108,8 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info)
FUNC_ENTER_NOAPI(FAIL)

/* Init io_info */
if (H5D__ioinfo_init(count, dset_info, &io_info) < 0)
if (H5D__ioinfo_init(count, H5D_IO_OP_READ, dset_info, &io_info) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize I/O info")
io_info.op_type = H5D_IO_OP_READ;

/* Allocate store buffer if necessary */
if (count > 1)
Expand Down Expand Up @@ -360,7 +360,7 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info)
if (NULL == (io_info.rbufs = H5MM_malloc(io_info.piece_count * sizeof(void *))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL,
"memory allocation failed for read buffer list")
if (io_info.tconv_buf)
if (io_info.max_tconv_type_size > 0)
if (NULL ==
(io_info.sel_pieces = H5MM_malloc(io_info.piece_count * sizeof(io_info.sel_pieces[0]))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL,
Expand Down Expand Up @@ -388,7 +388,7 @@ H5D__read(size_t count, H5D_dset_io_info_t *dset_info)
* (if using selection I/O and either multi dataset or type conversion) */
if (!H5D_LAYOUT_CB_PERFORM_IO(&io_info)) {
/* Check for type conversion */
if (io_info.tconv_buf) {
if (io_info.max_tconv_type_size > 0) {
/* Type conversion pathway */
if (H5D__scatgath_read_select(&io_info) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "type conversion selection read failed")
Expand Down Expand Up @@ -505,9 +505,8 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info)
FUNC_ENTER_NOAPI(FAIL)

/* Init io_info */
if (H5D__ioinfo_init(count, dset_info, &io_info) < 0)
if (H5D__ioinfo_init(count, H5D_IO_OP_WRITE, dset_info, &io_info) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize I/O info")
io_info.op_type = H5D_IO_OP_WRITE;

/* Allocate store buffer if necessary */
if (count > 1)
Expand Down Expand Up @@ -765,7 +764,7 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info)
if (NULL == (io_info.wbufs = H5MM_malloc(io_info.piece_count * sizeof(const void *))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL,
"memory allocation failed for write buffer list")
if (io_info.tconv_buf)
if (io_info.max_tconv_type_size > 0)
if (NULL ==
(io_info.sel_pieces = H5MM_malloc(io_info.piece_count * sizeof(io_info.sel_pieces[0]))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL,
Expand All @@ -791,7 +790,7 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info)
* (if using selection I/O and either multi dataset or type conversion) */
if (!H5D_LAYOUT_CB_PERFORM_IO(&io_info)) {
/* Check for type conversion */
if (io_info.tconv_buf) {
if (io_info.max_tconv_type_size > 0) {
/* Type conversion pathway */
if (H5D__scatgath_write_select(&io_info) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "type conversion selection write failed")
Expand Down Expand Up @@ -894,7 +893,8 @@ H5D__write(size_t count, H5D_dset_io_info_t *dset_info)
*-------------------------------------------------------------------------
*/
static herr_t
H5D__ioinfo_init(size_t count, H5D_dset_io_info_t *dset_info, H5D_io_info_t *io_info)
H5D__ioinfo_init(size_t count, H5D_io_op_type_t op_type, H5D_dset_io_info_t *dset_info,
H5D_io_info_t *io_info)
{
H5D_selection_io_mode_t selection_io_mode;

Expand All @@ -910,8 +910,9 @@ H5D__ioinfo_init(size_t count, H5D_dset_io_info_t *dset_info, H5D_io_info_t *io_
HDmemset(io_info, 0, sizeof(*io_info));

/* Set up simple fields */
io_info->f_sh = count > 0 ? H5F_SHARED(dset_info[0].dset->oloc.file) : NULL;
io_info->count = count;
io_info->op_type = op_type;
io_info->f_sh = count > 0 ? H5F_SHARED(dset_info[0].dset->oloc.file) : NULL;
io_info->count = count;

/* Start without multi-dataset I/O ops. If we're not using the collective
* I/O path then we will call the single dataset callbacks in a loop. */
Expand All @@ -927,7 +928,7 @@ H5D__ioinfo_init(size_t count, H5D_dset_io_info_t *dset_info, H5D_io_info_t *io_

/* Record no selection I/O cause if it was disabled by the API */
if (selection_io_mode == H5D_SELECTION_IO_MODE_OFF)
io_info->no_selection_io_cause = H5D_DISABLE_BY_API;
io_info->no_selection_io_cause = H5D_SEL_IO_DISABLE_BY_API;

#ifdef H5_HAVE_PARALLEL

Expand All @@ -936,6 +937,9 @@ H5D__ioinfo_init(size_t count, H5D_dset_io_info_t *dset_info, H5D_io_info_t *io_
io_info->using_mpi_vfd = H5F_HAS_FEATURE(dset_info[0].dset->oloc.file, H5FD_FEAT_HAS_MPI);
#endif /* H5_HAVE_PARALLEL */

/* Check if we could potentially use in-place type conversion. For now just check if it's a read op */
io_info->may_use_in_place_tconv = (op_type == H5D_IO_OP_READ);

FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5D__ioinfo_init() */

Expand Down Expand Up @@ -1144,42 +1148,40 @@ H5D__typeinfo_init_phase2(H5D_io_info_t *io_info)
size_t max_temp_buf; /* Maximum temporary buffer size */
size_t i; /* Local index variable */

/* Collective I/O, conversion buffer must be large enough for entire I/O (for now).
* Stick with individual background buffers */
/* Collective I/O, conversion buffer must be large enough for entire I/O (for now) */

/* Calculate size of buffers */
/* Calculate size of background buffer (tconv buf size was calculated in layout io_init callbacks)
*/
for (i = 0; i < io_info->count; i++) {
H5D_type_info_t *type_info = &io_info->dsets_info[i].type_info;

/* Check if type conversion buffer is needed for this dataset */
if (!type_info->is_conv_noop || !type_info->is_xform_noop) {
/* Add size of this dataset's type conversion buffer to the global type conversion buffer
* size */
io_info->tconv_buf_size += io_info->dsets_info[i].nelmts *
MAX(type_info->src_type_size, type_info->dst_type_size);

/* Check for background buffer */
if (type_info->need_bkg) {
/* Add size of this dataset's background buffer to the global background buffer size
*/
io_info->bkg_buf_size += io_info->dsets_info[i].nelmts * type_info->dst_type_size;

/* Check if we need to fill the background buffer with the destination contents */
if (type_info->need_bkg == H5T_BKG_YES)
io_info->must_fill_bkg = TRUE;
}
/* Check for background buffer */
if (type_info->need_bkg) {
/* Add size of this dataset's background buffer to the global background buffer size
*/
io_info->bkg_buf_size += io_info->dsets_info[i].nelmts * type_info->dst_type_size;

/* Check if we need to fill the background buffer with the destination contents */
if (type_info->need_bkg == H5T_BKG_YES)
io_info->must_fill_bkg = TRUE;
}
}

/* Get max temp buffer size from API context */
if (H5CX_get_max_temp_buf(&max_temp_buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't retrieve max. temp. buf size")

/* Check if the needed type conversion size is too big */
HDassert(io_info->bkg_buf_size <= io_info->tconv_buf_size);
/* Check if the needed type conversion or background buffer size is too big */
if (io_info->tconv_buf_size > max_temp_buf) {
io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF;
io_info->no_selection_io_cause |= H5D_TCONV_BUF_TOO_SMALL;
io_info->no_selection_io_cause |= H5D_SEL_IO_TCONV_BUF_TOO_SMALL;
io_info->tconv_buf_size = 0;
io_info->bkg_buf_size = 0;
io_info->must_fill_bkg = FALSE;
}
if (io_info->bkg_buf_size > max_temp_buf) {
io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF;
io_info->no_selection_io_cause |= H5D_SEL_IO_BKG_BUF_TOO_SMALL;
io_info->tconv_buf_size = 0;
io_info->bkg_buf_size = 0;
io_info->must_fill_bkg = FALSE;
Expand Down Expand Up @@ -1334,7 +1336,7 @@ H5D__ioinfo_adjust(H5D_io_info_t *io_info)
io_info->use_select_io = H5D_SELECTION_IO_MODE_ON;
else {
io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF;
io_info->no_selection_io_cause |= H5D_NO_VECTOR_OR_SELECTION_IO_CB;
io_info->no_selection_io_cause |= H5D_SEL_IO_DEFAULT_OFF;
}
}

Expand Down Expand Up @@ -1423,7 +1425,7 @@ H5D__typeinfo_init_phase3(H5D_io_info_t *io_info)
* assumes the tconv buf is large enough to hold all data in the I/O, and we don't want
* to impose this when there's no benefit */
io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF;
io_info->no_selection_io_cause |= H5D_DATATYPE_CONVERSION;
io_info->no_selection_io_cause |= H5D_SEL_IO_DATATYPE_CONVERSION;

/* Get max buffer size from API context */
if (H5CX_get_max_temp_buf(&max_temp_buf) < 0)
Expand Down
8 changes: 3 additions & 5 deletions src/H5Dmpio.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -739,10 +739,8 @@ H5D__mpio_opt_possible(H5D_io_info_t *io_info)
if (io_info->use_select_io == H5D_SELECTION_IO_MODE_DEFAULT) {
/* If the only reason(s) we've disabled collective are type conversions and/or transforms, enable
* selection I/O and re-enable collective I/O since it's supported by selection I/O */
if (global_cause[0] &&
!(global_cause[0] &
~((unsigned)H5D_MPIO_DATATYPE_CONVERSION | (unsigned)H5D_MPIO_DATA_TRANSFORMS)) &&
(io_info->use_select_io == H5D_SELECTION_IO_MODE_DEFAULT)) {
if (global_cause[0] && !(global_cause[0] & ~((unsigned)H5D_MPIO_DATATYPE_CONVERSION |
(unsigned)H5D_MPIO_DATA_TRANSFORMS))) {
HDassert(!(local_cause[0] &
~((unsigned)H5D_MPIO_DATATYPE_CONVERSION | (unsigned)H5D_MPIO_DATA_TRANSFORMS)));
local_cause[0] = 0;
Expand All @@ -752,7 +750,7 @@ H5D__mpio_opt_possible(H5D_io_info_t *io_info)
else {
/* Otherwise, there's currently no benefit to selection I/O, so leave it off */
io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF;
io_info->no_selection_io_cause |= H5D_NO_BENEFIT_BY_MPIO;
io_info->no_selection_io_cause |= H5D_SEL_IO_DEFAULT_OFF;
}
}

Expand Down
Loading

0 comments on commit c63610b

Please sign in to comment.