From 29e5e500f329cb40f76e0e257794332c9a0030f6 Mon Sep 17 00:00:00 2001 From: Dmitry Savitskiy Date: Sat, 17 Feb 2024 18:19:36 +0300 Subject: [PATCH] blobstore: dynamic allocation of request set object pool With large number of volumes and/or snapshots, a blobstore may run out of "request set" objects that are used to perform metadata I/O. The "request sets" are allocated from per-channel object pools, which are limited by 512 items by default. This effectively limits number of simultaneous metadata I/O requests. This commit enables a dynamic growth of per-channel "request set" pools. Signed-off-by: Dmitry Savitskiy --- lib/blob/blobstore.c | 20 ++----- lib/blob/blobstore.h | 3 +- lib/blob/request.c | 132 +++++++++++++++++++++++++++++++++++-------- lib/blob/request.h | 10 ++++ 4 files changed, 123 insertions(+), 42 deletions(-) diff --git a/lib/blob/blobstore.c b/lib/blob/blobstore.c index 8ea4ccfd386..972d647269e 100644 --- a/lib/blob/blobstore.c +++ b/lib/blob/blobstore.c @@ -3402,29 +3402,17 @@ bs_channel_create(void *io_device, void *ctx_buf) struct spdk_blob_store *bs = io_device; struct spdk_bs_channel *channel = ctx_buf; struct spdk_bs_dev *dev; - uint32_t max_ops = bs->max_channel_ops; - uint32_t i; dev = bs->dev; - channel->req_mem = calloc(max_ops, sizeof(struct spdk_bs_request_set)); - if (!channel->req_mem) { - return -1; - } - - TAILQ_INIT(&channel->reqs); - - for (i = 0; i < max_ops; i++) { - TAILQ_INSERT_TAIL(&channel->reqs, &channel->req_mem[i], link); - } - channel->bs = bs; channel->dev = dev; + bs_request_set_pool_init(&channel->req_pool); channel->dev_channel = dev->create_channel(dev); if (!channel->dev_channel) { SPDK_ERRLOG("Failed to create device channel.\n"); - free(channel->req_mem); + bs_request_set_pool_free(&channel->req_pool); return -1; } @@ -3432,7 +3420,7 @@ bs_channel_create(void *io_device, void *ctx_buf) SPDK_MALLOC_DMA); if (!channel->new_cluster_page) { SPDK_ERRLOG("Failed to allocate new cluster page\n"); - free(channel->req_mem); + bs_request_set_pool_free(&channel->req_pool); channel->dev->destroy_channel(channel->dev, channel->dev_channel); return -1; } @@ -3464,7 +3452,7 @@ bs_channel_destroy(void *io_device, void *ctx_buf) blob_esnap_destroy_bs_channel(channel); - free(channel->req_mem); + bs_request_set_pool_free(&channel->req_pool); spdk_free(channel->new_cluster_page); channel->dev->destroy_channel(channel->dev, channel->dev_channel); } diff --git a/lib/blob/blobstore.h b/lib/blob/blobstore.h index 8ab9cc7b3e2..5f4188983a3 100644 --- a/lib/blob/blobstore.h +++ b/lib/blob/blobstore.h @@ -203,8 +203,7 @@ struct spdk_blob_store { }; struct spdk_bs_channel { - struct spdk_bs_request_set *req_mem; - TAILQ_HEAD(, spdk_bs_request_set) reqs; + struct spdk_bs_request_set_pool req_pool; struct spdk_blob_store *bs; diff --git a/lib/blob/request.c b/lib/blob/request.c index 6e84015e769..e6cbfa296fa 100644 --- a/lib/blob/request.c +++ b/lib/blob/request.c @@ -14,6 +14,105 @@ #include "spdk/log.h" +void +bs_request_set_pool_init(struct spdk_bs_request_set_pool *req_pool) +{ + req_pool->req_mem = NULL; + TAILQ_INIT(&req_pool->reqs); +} + +void +bs_request_set_pool_free(struct spdk_bs_request_set_pool *req_pool) +{ + struct spdk_bs_request_set **ps = req_pool->req_mem; + + if (ps != NULL) { + while (*ps != NULL) { + free(*ps); + ++ps; + } + free(req_pool->req_mem); + req_pool->req_mem = NULL; + } +} + +static int +bs_request_set_pool_grow(struct spdk_bs_request_set_pool *req_pool, + uint32_t max_ops) +{ + struct spdk_bs_request_set **ps; + struct spdk_bs_request_set *new_set; + uint32_t i; + + /* Allocate a new chunk of spdk_bs_request_set elements */ + new_set = calloc(max_ops, sizeof(struct spdk_bs_request_set)); + if (!new_set) { + return -ENOMEM; + } + + /* Count the number of existing chunks and grow it by 1 */ + i = 0; + if (req_pool->req_mem != NULL) { + ps = req_pool->req_mem; + while (*ps++ != NULL) { + ++i; + } + } + ++i; + + ps = realloc(req_pool->req_mem, (i + 1) * sizeof(struct spdk_bs_request_set *)); + if (!ps) { + free(new_set); + return -ENOMEM; + } + ps[i - 1] = new_set; + ps[i] = NULL; /* Final NULL element indicates end of the list */ + req_pool->req_mem = ps; + + /* Append the elements from the new chunk to the list of free requests */ + for (i = 0; i < max_ops; i++) { + TAILQ_INSERT_TAIL(&req_pool->reqs, &new_set[i], link); + } + + return 0; +} + +static struct spdk_bs_request_set * +bs_request_set_alloc(struct spdk_bs_channel *channel, struct spdk_bs_cpl *cpl, + struct spdk_io_channel *back_channel) +{ + struct spdk_bs_request_set *set = NULL; + + if (channel->req_pool.req_mem != NULL) { + set = TAILQ_FIRST(&channel->req_pool.reqs); + } + + if (!set) { + if (bs_request_set_pool_grow(&channel->req_pool, + channel->bs->max_channel_ops) < 0) { + SPDK_ERRLOG("Failed to allocate per-channel request set pool\n"); + return NULL; + } + } + + set = TAILQ_FIRST(&channel->req_pool.reqs); + assert(set); + TAILQ_REMOVE(&channel->req_pool.reqs, set, link); + + set->cpl = *cpl; + set->bserrno = 0; + set->channel = channel; + set->back_channel = back_channel; + + return set; +} + +static inline void +bs_request_set_dealloc(struct spdk_bs_request_set *set) +{ + TAILQ_INSERT_TAIL(&set->channel->req_pool.reqs, set, link); +} + void bs_call_cpl(struct spdk_bs_cpl *cpl, int bserrno) { @@ -58,8 +157,7 @@ bs_request_set_complete(struct spdk_bs_request_set *set) struct spdk_bs_cpl cpl = set->cpl; int bserrno = set->bserrno; - TAILQ_INSERT_TAIL(&set->channel->reqs, set, link); - + bs_request_set_dealloc(set); bs_call_cpl(&cpl, bserrno); } @@ -81,16 +179,11 @@ bs_sequence_start(struct spdk_io_channel *_channel, struct spdk_bs_cpl *cpl, channel = spdk_io_channel_get_ctx(_channel); assert(channel != NULL); - set = TAILQ_FIRST(&channel->reqs); + + set = bs_request_set_alloc(channel, cpl, back_channel); if (!set) { return NULL; } - TAILQ_REMOVE(&channel->reqs, set, link); - - set->cpl = *cpl; - set->bserrno = 0; - set->channel = channel; - set->back_channel = back_channel; set->cb_args.cb_fn = bs_sequence_completion; set->cb_args.cb_arg = set; @@ -340,16 +433,11 @@ bs_batch_open(struct spdk_io_channel *_channel, struct spdk_bs_cpl *cpl, struct channel = spdk_io_channel_get_ctx(_channel); assert(channel != NULL); - set = TAILQ_FIRST(&channel->reqs); + + set = bs_request_set_alloc(channel, cpl, back_channel); if (!set) { return NULL; } - TAILQ_REMOVE(&channel->reqs, set, link); - - set->cpl = *cpl; - set->bserrno = 0; - set->channel = channel; - set->back_channel = back_channel; set->u.batch.cb_fn = NULL; set->u.batch.cb_arg = NULL; @@ -477,15 +565,11 @@ bs_user_op_alloc(struct spdk_io_channel *_channel, struct spdk_bs_cpl *cpl, channel = spdk_io_channel_get_ctx(_channel); assert(channel != NULL); - set = TAILQ_FIRST(&channel->reqs); + + set = bs_request_set_alloc(channel, cpl, NULL); if (!set) { return NULL; } - TAILQ_REMOVE(&channel->reqs, set, link); - - set->cpl = *cpl; - set->channel = channel; - set->back_channel = NULL; set->ext_io_opts = NULL; args = &set->u.user_op; @@ -541,7 +625,7 @@ bs_user_op_execute(spdk_bs_user_op_t *op) set->ext_io_opts); break; } - TAILQ_INSERT_TAIL(&set->channel->reqs, set, link); + bs_request_set_dealloc(set); } void @@ -552,7 +636,7 @@ bs_user_op_abort(spdk_bs_user_op_t *op, int bserrno) set = (struct spdk_bs_request_set *)op; set->cpl.u.blob_basic.cb_fn(set->cpl.u.blob_basic.cb_arg, bserrno); - TAILQ_INSERT_TAIL(&set->channel->reqs, set, link); + bs_request_set_dealloc(set); } SPDK_LOG_REGISTER_COMPONENT(blob_rw) diff --git a/lib/blob/request.h b/lib/blob/request.h index c1204fd8884..13cc0b82f22 100644 --- a/lib/blob/request.h +++ b/lib/blob/request.h @@ -127,6 +127,16 @@ struct spdk_bs_request_set { TAILQ_ENTRY(spdk_bs_request_set) link; }; +/* A pool of pre-allocated requested sets. */ +struct spdk_bs_request_set_pool { + struct spdk_bs_request_set **req_mem; + TAILQ_HEAD(, spdk_bs_request_set) reqs; +}; + +void bs_request_set_pool_init(struct spdk_bs_request_set_pool *req_pool); + +void bs_request_set_pool_free(struct spdk_bs_request_set_pool *req_pool); + void bs_call_cpl(struct spdk_bs_cpl *cpl, int bserrno); spdk_bs_sequence_t *bs_sequence_start_bs(struct spdk_io_channel *channel,