Skip to content

Commit

Permalink
io_uring: ensure io_queue_deferred() is out-of-line
Browse files Browse the repository at this point in the history
This is not the hot path, it's a slow path. Yet the locking for it is
in the hot path, and __cold does not prevent it from being inlined.

Move the locking to the function itself, and mark it noinline as well
to avoid it polluting the icache of the hot path.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
  • Loading branch information
Jens Axboe committed Dec 31, 2024
1 parent c5f7191 commit d62c2f0
Showing 1 changed file with 4 additions and 5 deletions.
9 changes: 4 additions & 5 deletions io_uring/io_uring.c
Original file line number Diff line number Diff line change
Expand Up @@ -550,8 +550,9 @@ void io_req_queue_iowq(struct io_kiocb *req)
io_req_task_work_add(req);
}

static __cold void io_queue_deferred(struct io_ring_ctx *ctx)
static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx)
{
spin_lock(&ctx->completion_lock);
while (!list_empty(&ctx->defer_list)) {
struct io_defer_entry *de = list_first_entry(&ctx->defer_list,
struct io_defer_entry, list);
Expand All @@ -562,6 +563,7 @@ static __cold void io_queue_deferred(struct io_ring_ctx *ctx)
io_req_task_queue(de->req);
kfree(de);
}
spin_unlock(&ctx->completion_lock);
}

void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
Expand All @@ -570,11 +572,8 @@ void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
io_poll_wq_wake(ctx);
if (ctx->off_timeout_used)
io_flush_timeouts(ctx);
if (ctx->drain_active) {
spin_lock(&ctx->completion_lock);
if (ctx->drain_active)
io_queue_deferred(ctx);
spin_unlock(&ctx->completion_lock);
}
if (ctx->has_evfd)
io_eventfd_flush_signal(ctx);
}
Expand Down

0 comments on commit d62c2f0

Please sign in to comment.