Skip to content

Commit

Permalink
io_uring: remove io_identity
Browse files Browse the repository at this point in the history 
We are no longer grabbing state, so no need to maintain an IO identity
that we COW if there are changes.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
  • Loading branch information
Jens Axboe committed Feb 22, 2021
1 parent 44526be commit 4379bf8
Show file tree
Hide file tree
Showing 4 changed files with 52 additions and 99 deletions.
26 changes: 26 additions & 0 deletions fs/io-wq.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@ struct io_worker {
struct io_wq_work *cur_work;
spinlock_t lock;

const struct cred *cur_creds;
const struct cred *saved_creds;

struct rcu_head rcu;
};

Expand Down Expand Up @@ -171,6 +174,11 @@ static void io_worker_exit(struct io_worker *worker)
worker->flags = 0;
preempt_enable();

if (worker->saved_creds) {
revert_creds(worker->saved_creds);
worker->cur_creds = worker->saved_creds = NULL;
}

raw_spin_lock_irq(&wqe->lock);
hlist_nulls_del_rcu(&worker->nulls_node);
list_del_rcu(&worker->all_list);
Expand Down Expand Up @@ -312,6 +320,10 @@ static void __io_worker_idle(struct io_wqe *wqe, struct io_worker *worker)
worker->flags |= IO_WORKER_F_FREE;
hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list);
}
if (worker->saved_creds) {
revert_creds(worker->saved_creds);
worker->cur_creds = worker->saved_creds = NULL;
}
}

static inline unsigned int io_get_work_hash(struct io_wq_work *work)
Expand Down Expand Up @@ -359,6 +371,18 @@ static void io_flush_signals(void)
}
}

static void io_wq_switch_creds(struct io_worker *worker,
struct io_wq_work *work)
{
const struct cred *old_creds = override_creds(work->creds);

worker->cur_creds = work->creds;
if (worker->saved_creds)
put_cred(old_creds); /* creds set by previous switch */
else
worker->saved_creds = old_creds;
}

static void io_assign_current_work(struct io_worker *worker,
struct io_wq_work *work)
{
Expand Down Expand Up @@ -407,6 +431,8 @@ static void io_worker_handle_work(struct io_worker *worker)
unsigned int hash = io_get_work_hash(work);

next_hashed = wq_next_work(work);
if (work->creds && worker->cur_creds != work->creds)
io_wq_switch_creds(worker, work);
wq->do_work(work);
io_assign_current_work(worker, NULL);

Expand Down
2 changes: 1 addition & 1 deletion fs/io-wq.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ static inline void wq_list_del(struct io_wq_work_list *list,

struct io_wq_work {
struct io_wq_work_node list;
struct io_identity *identity;
const struct cred *creds;
unsigned flags;
};

Expand Down
104 changes: 25 additions & 79 deletions fs/io_uring.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1094,7 +1094,7 @@ static bool io_match_task(struct io_kiocb *head,
continue;
if (req->file && req->file->f_op == &io_uring_fops)
return true;
if (req->work.identity->files == files)
if (req->task->files == files)
return true;
}
return false;
Expand Down Expand Up @@ -1218,31 +1218,6 @@ static inline void req_set_fail_links(struct io_kiocb *req)
req->flags |= REQ_F_FAIL_LINK;
}

/*
* None of these are dereferenced, they are simply used to check if any of
* them have changed. If we're under current and check they are still the
* same, we're fine to grab references to them for actual out-of-line use.
*/
static void io_init_identity(struct io_identity *id)
{
id->files = current->files;
id->mm = current->mm;
#ifdef CONFIG_BLK_CGROUP
rcu_read_lock();
id->blkcg_css = blkcg_css();
rcu_read_unlock();
#endif
id->creds = current_cred();
id->nsproxy = current->nsproxy;
id->fs = current->fs;
id->fsize = rlimit(RLIMIT_FSIZE);
#ifdef CONFIG_AUDIT
id->loginuid = current->loginuid;
id->sessionid = current->sessionid;
#endif
refcount_set(&id->count, 1);
}

static inline void __io_req_init_async(struct io_kiocb *req)
{
memset(&req->work, 0, sizeof(req->work));
Expand All @@ -1255,17 +1230,10 @@ static inline void __io_req_init_async(struct io_kiocb *req)
*/
static inline void io_req_init_async(struct io_kiocb *req)
{
struct io_uring_task *tctx = current->io_uring;

if (req->flags & REQ_F_WORK_INITIALIZED)
return;

__io_req_init_async(req);

/* Grab a ref if this isn't our static identity */
req->work.identity = tctx->identity;
if (tctx->identity != &tctx->__identity)
refcount_inc(&req->work.identity->count);
}

static void io_ring_ctx_ref_free(struct percpu_ref *ref)
Expand Down Expand Up @@ -1350,19 +1318,15 @@ static bool req_need_defer(struct io_kiocb *req, u32 seq)
return false;
}

static void io_put_identity(struct io_uring_task *tctx, struct io_kiocb *req)
{
if (req->work.identity == &tctx->__identity)
return;
if (refcount_dec_and_test(&req->work.identity->count))
kfree(req->work.identity);
}

static void io_req_clean_work(struct io_kiocb *req)
{
if (!(req->flags & REQ_F_WORK_INITIALIZED))
return;

if (req->work.creds) {
put_cred(req->work.creds);
req->work.creds = NULL;
}
if (req->flags & REQ_F_INFLIGHT) {
struct io_ring_ctx *ctx = req->ctx;
struct io_uring_task *tctx = req->task->io_uring;
Expand All @@ -1377,7 +1341,6 @@ static void io_req_clean_work(struct io_kiocb *req)
}

req->flags &= ~REQ_F_WORK_INITIALIZED;
io_put_identity(req->task->io_uring, req);
}

static void io_req_track_inflight(struct io_kiocb *req)
Expand Down Expand Up @@ -1411,6 +1374,8 @@ static void io_prep_async_work(struct io_kiocb *req)
if (def->unbound_nonreg_file)
req->work.flags |= IO_WQ_WORK_UNBOUND;
}
if (!req->work.creds)
req->work.creds = get_current_cred();
}

static void io_prep_async_link(struct io_kiocb *req)
Expand Down Expand Up @@ -6376,9 +6341,9 @@ static void __io_queue_sqe(struct io_kiocb *req)
const struct cred *old_creds = NULL;
int ret;

if ((req->flags & REQ_F_WORK_INITIALIZED) &&
req->work.identity->creds != current_cred())
old_creds = override_creds(req->work.identity->creds);
if ((req->flags & REQ_F_WORK_INITIALIZED) && req->work.creds &&
req->work.creds != current_cred())
old_creds = override_creds(req->work.creds);

ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER);

Expand Down Expand Up @@ -6508,16 +6473,11 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,

id = READ_ONCE(sqe->personality);
if (id) {
struct io_identity *iod;

iod = idr_find(&ctx->personality_idr, id);
if (unlikely(!iod))
return -EINVAL;
refcount_inc(&iod->count);

__io_req_init_async(req);
get_cred(iod->creds);
req->work.identity = iod;
req->work.creds = idr_find(&ctx->personality_idr, id);
if (unlikely(!req->work.creds))
return -EINVAL;
get_cred(req->work.creds);
}

state = &ctx->submit_state;
Expand Down Expand Up @@ -7936,8 +7896,6 @@ static int io_uring_alloc_task_context(struct task_struct *task,
tctx->last = NULL;
atomic_set(&tctx->in_idle, 0);
tctx->sqpoll = false;
io_init_identity(&tctx->__identity);
tctx->identity = &tctx->__identity;
task->io_uring = tctx;
spin_lock_init(&tctx->task_lock);
INIT_WQ_LIST(&tctx->task_list);
Expand All @@ -7951,9 +7909,6 @@ void __io_uring_free(struct task_struct *tsk)
struct io_uring_task *tctx = tsk->io_uring;

WARN_ON_ONCE(!xa_empty(&tctx->xa));
WARN_ON_ONCE(refcount_read(&tctx->identity->count) != 1);
if (tctx->identity != &tctx->__identity)
kfree(tctx->identity);
percpu_counter_destroy(&tctx->inflight);
kfree(tctx);
tsk->io_uring = NULL;
Expand Down Expand Up @@ -8593,13 +8548,11 @@ static int io_uring_fasync(int fd, struct file *file, int on)

static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
{
struct io_identity *iod;
const struct cred *creds;

iod = idr_remove(&ctx->personality_idr, id);
if (iod) {
put_cred(iod->creds);
if (refcount_dec_and_test(&iod->count))
kfree(iod);
creds = idr_remove(&ctx->personality_idr, id);
if (creds) {
put_cred(creds);
return 0;
}

Expand Down Expand Up @@ -9300,8 +9253,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
#ifdef CONFIG_PROC_FS
static int io_uring_show_cred(int id, void *p, void *data)
{
struct io_identity *iod = p;
const struct cred *cred = iod->creds;
const struct cred *cred = p;
struct seq_file *m = data;
struct user_namespace *uns = seq_user_ns(m);
struct group_info *gi;
Expand Down Expand Up @@ -9732,21 +9684,15 @@ static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)

static int io_register_personality(struct io_ring_ctx *ctx)
{
struct io_identity *id;
const struct cred *creds;
int ret;

id = kmalloc(sizeof(*id), GFP_KERNEL);
if (unlikely(!id))
return -ENOMEM;

io_init_identity(id);
id->creds = get_current_cred();
creds = get_current_cred();

ret = idr_alloc_cyclic(&ctx->personality_idr, id, 1, USHRT_MAX, GFP_KERNEL);
if (ret < 0) {
put_cred(id->creds);
kfree(id);
}
ret = idr_alloc_cyclic(&ctx->personality_idr, (void *) creds, 1,
USHRT_MAX, GFP_KERNEL);
if (ret < 0)
put_cred(creds);
return ret;
}

Expand Down
19 changes: 0 additions & 19 deletions include/linux/io_uring.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,23 +5,6 @@
#include <linux/sched.h>
#include <linux/xarray.h>

struct io_identity {
struct files_struct *files;
struct mm_struct *mm;
#ifdef CONFIG_BLK_CGROUP
struct cgroup_subsys_state *blkcg_css;
#endif
const struct cred *creds;
struct nsproxy *nsproxy;
struct fs_struct *fs;
unsigned long fsize;
#ifdef CONFIG_AUDIT
kuid_t loginuid;
unsigned int sessionid;
#endif
refcount_t count;
};

struct io_wq_work_node {
struct io_wq_work_node *next;
};
Expand All @@ -38,8 +21,6 @@ struct io_uring_task {
struct file *last;
void *io_wq;
struct percpu_counter inflight;
struct io_identity __identity;
struct io_identity *identity;
atomic_t in_idle;
bool sqpoll;

Expand Down

0 comments on commit 4379bf8

Please sign in to comment.