Skip to content

Commit

Permalink
io_uring: propagate issue_flags state down to file assignment
Browse files Browse the repository at this point in the history 
We'll need this in a future patch, when we could be assigning the file
after the prep stage. While at it, get rid of the io_file_get() helper,
it just makes the code harder to read.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
  • Loading branch information
Jens Axboe committed Apr 7, 2022
1 parent 584b018 commit 5106dd6
Showing 1 changed file with 47 additions and 35 deletions.
82 changes: 47 additions & 35 deletions fs/io_uring.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1183,8 +1183,9 @@ static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
struct io_uring_rsrc_update2 *up,
unsigned nr_args);
static void io_clean_op(struct io_kiocb *req);
static struct file *io_file_get(struct io_ring_ctx *ctx,
struct io_kiocb *req, int fd, bool fixed);
static inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
unsigned issue_flags);
static inline struct file *io_file_get_normal(struct io_kiocb *req, int fd);
static void __io_queue_sqe(struct io_kiocb *req);
static void io_rsrc_put_work(struct work_struct *work);

Expand Down Expand Up @@ -1314,13 +1315,20 @@ static void io_rsrc_refs_refill(struct io_ring_ctx *ctx)
}

static inline void io_req_set_rsrc_node(struct io_kiocb *req,
struct io_ring_ctx *ctx)
struct io_ring_ctx *ctx,
unsigned int issue_flags)
{
if (!req->fixed_rsrc_refs) {
req->fixed_rsrc_refs = &ctx->rsrc_node->refs;
ctx->rsrc_cached_refs--;
if (unlikely(ctx->rsrc_cached_refs < 0))
io_rsrc_refs_refill(ctx);

if (!(issue_flags & IO_URING_F_UNLOCKED)) {
lockdep_assert_held(&ctx->uring_lock);
ctx->rsrc_cached_refs--;
if (unlikely(ctx->rsrc_cached_refs < 0))
io_rsrc_refs_refill(ctx);
} else {
percpu_ref_get(req->fixed_rsrc_refs);
}
}
}

Expand Down Expand Up @@ -3330,7 +3338,8 @@ static int __io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter
return 0;
}

static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter)
static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter,
unsigned int issue_flags)
{
struct io_mapped_ubuf *imu = req->imu;
u16 index, buf_index = req->buf_index;
Expand All @@ -3340,7 +3349,7 @@ static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter)

if (unlikely(buf_index >= ctx->nr_user_bufs))
return -EFAULT;
io_req_set_rsrc_node(req, ctx);
io_req_set_rsrc_node(req, ctx, issue_flags);
index = array_index_nospec(buf_index, ctx->nr_user_bufs);
imu = READ_ONCE(ctx->user_bufs[index]);
req->imu = imu;
Expand Down Expand Up @@ -3502,7 +3511,7 @@ static struct iovec *__io_import_iovec(int rw, struct io_kiocb *req,
ssize_t ret;

if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) {
ret = io_import_fixed(req, rw, iter);
ret = io_import_fixed(req, rw, iter, issue_flags);
if (ret)
return ERR_PTR(ret);
return NULL;
Expand Down Expand Up @@ -4394,8 +4403,10 @@ static int io_tee(struct io_kiocb *req, unsigned int issue_flags)
if (issue_flags & IO_URING_F_NONBLOCK)
return -EAGAIN;

in = io_file_get(req->ctx, req, sp->splice_fd_in,
(sp->flags & SPLICE_F_FD_IN_FIXED));
if (sp->flags & SPLICE_F_FD_IN_FIXED)
in = io_file_get_fixed(req, sp->splice_fd_in, IO_URING_F_UNLOCKED);
else
in = io_file_get_normal(req, sp->splice_fd_in);
if (!in) {
ret = -EBADF;
goto done;
Expand Down Expand Up @@ -4434,8 +4445,10 @@ static int io_splice(struct io_kiocb *req, unsigned int issue_flags)
if (issue_flags & IO_URING_F_NONBLOCK)
return -EAGAIN;

in = io_file_get(req->ctx, req, sp->splice_fd_in,
(sp->flags & SPLICE_F_FD_IN_FIXED));
if (sp->flags & SPLICE_F_FD_IN_FIXED)
in = io_file_get_fixed(req, sp->splice_fd_in, IO_URING_F_UNLOCKED);
else
in = io_file_get_normal(req, sp->splice_fd_in);
if (!in) {
ret = -EBADF;
goto done;
Expand Down Expand Up @@ -5973,7 +5986,7 @@ static void io_poll_remove_entries(struct io_kiocb *req)
* either spurious wakeup or multishot CQE is served. 0 when it's done with
* the request, then the mask is stored in req->result.
*/
static int io_poll_check_events(struct io_kiocb *req)
static int io_poll_check_events(struct io_kiocb *req, bool locked)
{
struct io_ring_ctx *ctx = req->ctx;
struct io_poll_iocb *poll = io_poll_get_single(req);
Expand Down Expand Up @@ -6030,7 +6043,7 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked)
struct io_ring_ctx *ctx = req->ctx;
int ret;

ret = io_poll_check_events(req);
ret = io_poll_check_events(req, *locked);
if (ret > 0)
return;

Expand All @@ -6055,7 +6068,7 @@ static void io_apoll_task_func(struct io_kiocb *req, bool *locked)
struct io_ring_ctx *ctx = req->ctx;
int ret;

ret = io_poll_check_events(req);
ret = io_poll_check_events(req, *locked);
if (ret > 0)
return;

Expand Down Expand Up @@ -7460,46 +7473,43 @@ static void io_fixed_file_set(struct io_fixed_file *file_slot, struct file *file
file_slot->file_ptr = file_ptr;
}

static inline struct file *io_file_get_fixed(struct io_ring_ctx *ctx,
struct io_kiocb *req, int fd)
static inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
unsigned int issue_flags)
{
struct file *file;
struct io_ring_ctx *ctx = req->ctx;
struct file *file = NULL;
unsigned long file_ptr;

if (issue_flags & IO_URING_F_UNLOCKED)
mutex_lock(&ctx->uring_lock);

if (unlikely((unsigned int)fd >= ctx->nr_user_files))
return NULL;
goto out;
fd = array_index_nospec(fd, ctx->nr_user_files);
file_ptr = io_fixed_file_slot(&ctx->file_table, fd)->file_ptr;
file = (struct file *) (file_ptr & FFS_MASK);
file_ptr &= ~FFS_MASK;
/* mask in overlapping REQ_F and FFS bits */
req->flags |= (file_ptr << REQ_F_SUPPORT_NOWAIT_BIT);
io_req_set_rsrc_node(req, ctx);
io_req_set_rsrc_node(req, ctx, 0);
out:
if (issue_flags & IO_URING_F_UNLOCKED)
mutex_unlock(&ctx->uring_lock);
return file;
}

static struct file *io_file_get_normal(struct io_ring_ctx *ctx,
struct io_kiocb *req, int fd)
static struct file *io_file_get_normal(struct io_kiocb *req, int fd)
{
struct file *file = fget(fd);

trace_io_uring_file_get(ctx, req, req->user_data, fd);
trace_io_uring_file_get(req->ctx, req, req->user_data, fd);

/* we don't allow fixed io_uring files */
if (file && unlikely(file->f_op == &io_uring_fops))
io_req_track_inflight(req);
return file;
}

static inline struct file *io_file_get(struct io_ring_ctx *ctx,
struct io_kiocb *req, int fd, bool fixed)
{
if (fixed)
return io_file_get_fixed(ctx, req, fd);
else
return io_file_get_normal(ctx, req, fd);
}

static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked)
{
struct io_kiocb *prev = req->timeout.prev;
Expand Down Expand Up @@ -7749,8 +7759,10 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
blk_start_plug_nr_ios(&state->plug, state->submit_nr);
}

req->file = io_file_get(ctx, req, READ_ONCE(sqe->fd),
(sqe_flags & IOSQE_FIXED_FILE));
if (req->flags & REQ_F_FIXED_FILE)
req->file = io_file_get_fixed(req, READ_ONCE(sqe->fd), 0);
else
req->file = io_file_get_normal(req, READ_ONCE(sqe->fd));
if (unlikely(!req->file))
return -EBADF;
}
Expand Down

0 comments on commit 5106dd6

Please sign in to comment.