Skip to content

Commit

Permalink
io_uring: batch getting pcpu references
Browse files Browse the repository at this point in the history
percpu_ref_tryget() has its own overhead. Instead getting a reference
for each request, grab a bunch once per io_submit_sqes().

~5% throughput boost for a "submit and wait 128 nops" benchmark.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>

__io_req_free_empty() -> __io_req_do_free()

Signed-off-by: Jens Axboe <axboe@kernel.dk>
  • Loading branch information
Pavel Begunkov authored and Jens Axboe committed Jan 21, 2020
1 parent 4e5ef02 commit 2b85edf
Showing 1 changed file with 17 additions and 9 deletions.
26 changes: 17 additions & 9 deletions fs/io_uring.c
Original file line number Diff line number Diff line change
Expand Up @@ -1083,9 +1083,6 @@ static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
struct io_kiocb *req;

if (!percpu_ref_tryget(&ctx->refs))
return NULL;

if (!state) {
req = kmem_cache_alloc(req_cachep, gfp);
if (unlikely(!req))
Expand Down Expand Up @@ -1145,6 +1142,14 @@ static void io_free_req_many(struct io_ring_ctx *ctx, void **reqs, int *nr)
}
}

static void __io_req_do_free(struct io_kiocb *req)
{
if (likely(!io_is_fallback_req(req)))
kmem_cache_free(req_cachep, req);
else
clear_bit_unlock(0, (unsigned long *) req->ctx->fallback_req);
}

static void __io_free_req(struct io_kiocb *req)
{
struct io_ring_ctx *ctx = req->ctx;
Expand All @@ -1166,11 +1171,9 @@ static void __io_free_req(struct io_kiocb *req)
wake_up(&ctx->inflight_wait);
spin_unlock_irqrestore(&ctx->inflight_lock, flags);
}
percpu_ref_put(&ctx->refs);
if (likely(!io_is_fallback_req(req)))
kmem_cache_free(req_cachep, req);
else
clear_bit_unlock(0, (unsigned long *) ctx->fallback_req);

percpu_ref_put(&req->ctx->refs);
__io_req_do_free(req);
}

static bool io_link_cancel_timeout(struct io_kiocb *req)
Expand Down Expand Up @@ -4539,6 +4542,9 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
return -EBUSY;
}

if (!percpu_ref_tryget_many(&ctx->refs, nr))
return -EAGAIN;

if (nr > IO_PLUG_THRESHOLD) {
io_submit_state_start(&state, nr);
statep = &state;
Expand All @@ -4555,7 +4561,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
break;
}
if (!io_get_sqring(ctx, req, &sqe)) {
__io_free_req(req);
__io_req_do_free(req);
break;
}

Expand Down Expand Up @@ -4586,6 +4592,8 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
break;
}

if (submitted != nr)
percpu_ref_put_many(&ctx->refs, nr - submitted);
if (link)
io_queue_link_head(link);
if (statep)
Expand Down

0 comments on commit 2b85edf

Please sign in to comment.