Skip to content

Commit

Permalink
blkcg: move refcnt to blkcg core
Browse files Browse the repository at this point in the history 
Currently, blkcg policy implementations manage blkg refcnt duplicating
mostly identical code in both policies.  This patch moves refcnt to
blkg and let blkcg core handle refcnt and freeing of blkgs.

* cfq blkgs now also get freed via RCU.

* cfq blkgs lose RB_EMPTY_ROOT() sanity check on blkg free.  If
  necessary, we can add blkio_exit_group_fn() to resurrect this.

Signed-off-by: Tejun Heo <tj@kernel.org>
Cc: Vivek Goyal <vgoyal@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
  • Loading branch information
Tejun Heo authored and Jens Axboe committed Mar 6, 2012
1 parent 0381411 commit 1adaf3d
Show file tree
Hide file tree
Showing 4 changed files with 73 additions and 102 deletions.
24 changes: 24 additions & 0 deletions block/blk-cgroup.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -463,6 +463,7 @@ static struct blkio_group *blkg_alloc(struct blkio_cgroup *blkcg,
rcu_assign_pointer(blkg->q, q);
blkg->blkcg = blkcg;
blkg->plid = pol->plid;
blkg->refcnt = 1;
cgroup_path(blkcg->css.cgroup, blkg->path, sizeof(blkg->path));

/* alloc per-policy data */
Expand Down Expand Up @@ -633,6 +634,29 @@ void blkg_destroy_all(struct request_queue *q)
}
}

static void blkg_rcu_free(struct rcu_head *rcu_head)
{
blkg_free(container_of(rcu_head, struct blkio_group, rcu_head));
}

void __blkg_release(struct blkio_group *blkg)
{
/* release the extra blkcg reference this blkg has been holding */
css_put(&blkg->blkcg->css);

/*
* A group is freed in rcu manner. But having an rcu lock does not
* mean that one can access all the fields of blkg and assume these
* are valid. For example, don't try to follow throtl_data and
* request queue links.
*
* Having a reference to blkg under an rcu allows acess to only
* values local to groups like group stats and group rate limits
*/
call_rcu(&blkg->rcu_head, blkg_rcu_free);
}
EXPORT_SYMBOL_GPL(__blkg_release);

static void blkio_reset_stats_cpu(struct blkio_group *blkg)
{
struct blkio_group_stats_cpu *stats_cpu;
Expand Down
35 changes: 35 additions & 0 deletions block/blk-cgroup.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,8 @@ struct blkio_group {
char path[128];
/* policy which owns this blk group */
enum blkio_policy_id plid;
/* reference count */
int refcnt;

/* Configuration */
struct blkio_group_conf conf;
Expand All @@ -188,6 +190,8 @@ struct blkio_group {
struct blkio_group_stats_cpu __percpu *stats_cpu;

struct blkg_policy_data *pd;

struct rcu_head rcu_head;
};

typedef void (blkio_init_group_fn)(struct blkio_group *blkg);
Expand Down Expand Up @@ -272,6 +276,35 @@ static inline char *blkg_path(struct blkio_group *blkg)
return blkg->path;
}

/**
* blkg_get - get a blkg reference
* @blkg: blkg to get
*
* The caller should be holding queue_lock and an existing reference.
*/
static inline void blkg_get(struct blkio_group *blkg)
{
lockdep_assert_held(blkg->q->queue_lock);
WARN_ON_ONCE(!blkg->refcnt);
blkg->refcnt++;
}

void __blkg_release(struct blkio_group *blkg);

/**
* blkg_put - put a blkg reference
* @blkg: blkg to put
*
* The caller should be holding queue_lock.
*/
static inline void blkg_put(struct blkio_group *blkg)
{
lockdep_assert_held(blkg->q->queue_lock);
WARN_ON_ONCE(blkg->refcnt <= 0);
if (!--blkg->refcnt)
__blkg_release(blkg);
}

#else

struct blkio_group {
Expand All @@ -292,6 +325,8 @@ static inline void *blkg_to_pdata(struct blkio_group *blkg,
static inline struct blkio_group *pdata_to_blkg(void *pdata,
struct blkio_policy_type *pol) { return NULL; }
static inline char *blkg_path(struct blkio_group *blkg) { return NULL; }
static inline void blkg_get(struct blkio_group *blkg) { }
static inline void blkg_put(struct blkio_group *blkg) { }

#endif

Expand Down
58 changes: 4 additions & 54 deletions block/blk-throttle.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ struct throtl_grp {
*/
unsigned long disptime;

atomic_t ref;
unsigned int flags;

/* Two lists for READ and WRITE */
Expand All @@ -80,8 +79,6 @@ struct throtl_grp {

/* Some throttle limits got updated for the group */
int limits_changed;

struct rcu_head rcu_head;
};

struct throtl_data
Expand Down Expand Up @@ -151,45 +148,6 @@ static inline unsigned int total_nr_queued(struct throtl_data *td)
return td->nr_queued[0] + td->nr_queued[1];
}

static inline struct throtl_grp *throtl_ref_get_tg(struct throtl_grp *tg)
{
atomic_inc(&tg->ref);
return tg;
}

static void throtl_free_tg(struct rcu_head *head)
{
struct throtl_grp *tg = container_of(head, struct throtl_grp, rcu_head);
struct blkio_group *blkg = tg_to_blkg(tg);

free_percpu(blkg->stats_cpu);
kfree(blkg->pd);
kfree(blkg);
}

static void throtl_put_tg(struct throtl_grp *tg)
{
struct blkio_group *blkg = tg_to_blkg(tg);

BUG_ON(atomic_read(&tg->ref) <= 0);
if (!atomic_dec_and_test(&tg->ref))
return;

/* release the extra blkcg reference this blkg has been holding */
css_put(&blkg->blkcg->css);

/*
* A group is freed in rcu manner. But having an rcu lock does not
* mean that one can access all the fields of blkg and assume these
* are valid. For example, don't try to follow throtl_data and
* request queue links.
*
* Having a reference to blkg under an rcu allows acess to only
* values local to groups like group stats and group rate limits
*/
call_rcu(&tg->rcu_head, throtl_free_tg);
}

static void throtl_init_blkio_group(struct blkio_group *blkg)
{
struct throtl_grp *tg = blkg_to_tg(blkg);
Expand All @@ -204,14 +162,6 @@ static void throtl_init_blkio_group(struct blkio_group *blkg)
tg->bps[WRITE] = -1;
tg->iops[READ] = -1;
tg->iops[WRITE] = -1;

/*
* Take the initial reference that will be released on destroy
* This can be thought of a joint reference by cgroup and
* request queue which will be dropped by either request queue
* exit or cgroup deletion path depending on who is exiting first.
*/
atomic_set(&tg->ref, 1);
}

static void throtl_link_blkio_group(struct request_queue *q,
Expand Down Expand Up @@ -648,7 +598,7 @@ static void throtl_add_bio_tg(struct throtl_data *td, struct throtl_grp *tg,

bio_list_add(&tg->bio_lists[rw], bio);
/* Take a bio reference on tg */
throtl_ref_get_tg(tg);
blkg_get(tg_to_blkg(tg));
tg->nr_queued[rw]++;
td->nr_queued[rw]++;
throtl_enqueue_tg(td, tg);
Expand Down Expand Up @@ -681,8 +631,8 @@ static void tg_dispatch_one_bio(struct throtl_data *td, struct throtl_grp *tg,

bio = bio_list_pop(&tg->bio_lists[rw]);
tg->nr_queued[rw]--;
/* Drop bio reference on tg */
throtl_put_tg(tg);
/* Drop bio reference on blkg */
blkg_put(tg_to_blkg(tg));

BUG_ON(td->nr_queued[rw] <= 0);
td->nr_queued[rw]--;
Expand Down Expand Up @@ -880,7 +830,7 @@ throtl_destroy_tg(struct throtl_data *td, struct throtl_grp *tg)
* Put the reference taken at the time of creation so that when all
* queues are gone, group can be destroyed.
*/
throtl_put_tg(tg);
blkg_put(tg_to_blkg(tg));
td->nr_undestroyed_grps--;
}

Expand Down
58 changes: 10 additions & 48 deletions block/cfq-iosched.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,6 @@ struct cfq_group {
enum wl_prio_t saved_serving_prio;
#ifdef CONFIG_CFQ_GROUP_IOSCHED
struct hlist_node cfqd_node;
int ref;
#endif
/* number of requests that are on the dispatch list or inside driver */
int dispatched;
Expand Down Expand Up @@ -1071,14 +1070,6 @@ static void cfq_init_blkio_group(struct blkio_group *blkg)

cfq_init_cfqg_base(cfqg);
cfqg->weight = blkg->blkcg->weight;

/*
* Take the initial reference that will be released on destroy
* This can be thought of a joint reference by cgroup and
* elevator which will be dropped by either elevator exit
* or cgroup deletion path depending on who is exiting first.
*/
cfqg->ref = 1;
}

/*
Expand All @@ -1105,12 +1096,6 @@ static struct cfq_group *cfq_lookup_create_cfqg(struct cfq_data *cfqd,
return cfqg;
}

static inline struct cfq_group *cfq_ref_get_cfqg(struct cfq_group *cfqg)
{
cfqg->ref++;
return cfqg;
}

static void cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg)
{
/* Currently, all async queues are mapped to root group */
Expand All @@ -1119,28 +1104,7 @@ static void cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg)

cfqq->cfqg = cfqg;
/* cfqq reference on cfqg */
cfqq->cfqg->ref++;
}

static void cfq_put_cfqg(struct cfq_group *cfqg)
{
struct blkio_group *blkg = cfqg_to_blkg(cfqg);
struct cfq_rb_root *st;
int i, j;

BUG_ON(cfqg->ref <= 0);
cfqg->ref--;
if (cfqg->ref)
return;

/* release the extra blkcg reference this blkg has been holding */
css_put(&blkg->blkcg->css);

for_each_cfqg_st(cfqg, i, j, st)
BUG_ON(!RB_EMPTY_ROOT(&st->rb));
free_percpu(blkg->stats_cpu);
kfree(blkg->pd);
kfree(blkg);
blkg_get(cfqg_to_blkg(cfqg));
}

static void cfq_destroy_cfqg(struct cfq_data *cfqd, struct cfq_group *cfqg)
Expand All @@ -1157,7 +1121,7 @@ static void cfq_destroy_cfqg(struct cfq_data *cfqd, struct cfq_group *cfqg)
* Put the reference taken at the time of creation so that when all
* queues are gone, group can be destroyed.
*/
cfq_put_cfqg(cfqg);
blkg_put(cfqg_to_blkg(cfqg));
}

static bool cfq_release_cfq_groups(struct cfq_data *cfqd)
Expand Down Expand Up @@ -1225,18 +1189,12 @@ static struct cfq_group *cfq_lookup_create_cfqg(struct cfq_data *cfqd,
return cfqd->root_group;
}

static inline struct cfq_group *cfq_ref_get_cfqg(struct cfq_group *cfqg)
{
return cfqg;
}

static inline void
cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg) {
cfqq->cfqg = cfqg;
}

static void cfq_release_cfq_groups(struct cfq_data *cfqd) {}
static inline void cfq_put_cfqg(struct cfq_group *cfqg) {}

#endif /* GROUP_IOSCHED */

Expand Down Expand Up @@ -2630,7 +2588,7 @@ static void cfq_put_queue(struct cfq_queue *cfqq)

BUG_ON(cfq_cfqq_on_rr(cfqq));
kmem_cache_free(cfq_pool, cfqq);
cfq_put_cfqg(cfqg);
blkg_put(cfqg_to_blkg(cfqg));
}

static void cfq_put_cooperator(struct cfq_queue *cfqq)
Expand Down Expand Up @@ -3382,7 +3340,7 @@ static void cfq_put_request(struct request *rq)
cfqq->allocated[rw]--;

/* Put down rq reference on cfqg */
cfq_put_cfqg(RQ_CFQG(rq));
blkg_put(cfqg_to_blkg(RQ_CFQG(rq)));
rq->elv.priv[0] = NULL;
rq->elv.priv[1] = NULL;

Expand Down Expand Up @@ -3477,8 +3435,9 @@ cfq_set_request(struct request_queue *q, struct request *rq, gfp_t gfp_mask)
cfqq->allocated[rw]++;

cfqq->ref++;
blkg_get(cfqg_to_blkg(cfqq->cfqg));
rq->elv.priv[0] = cfqq;
rq->elv.priv[1] = cfq_ref_get_cfqg(cfqq->cfqg);
rq->elv.priv[1] = cfqq->cfqg;
spin_unlock_irq(q->queue_lock);
return 0;
}
Expand Down Expand Up @@ -3676,8 +3635,11 @@ static int cfq_init_queue(struct request_queue *q)
*/
cfq_init_cfqq(cfqd, &cfqd->oom_cfqq, 1, 0);
cfqd->oom_cfqq.ref++;

spin_lock_irq(q->queue_lock);
cfq_link_cfqq_cfqg(&cfqd->oom_cfqq, cfqd->root_group);
cfq_put_cfqg(cfqd->root_group);
blkg_put(cfqg_to_blkg(cfqd->root_group));
spin_unlock_irq(q->queue_lock);

init_timer(&cfqd->idle_slice_timer);
cfqd->idle_slice_timer.function = cfq_idle_slice_timer;
Expand Down

0 comments on commit 1adaf3d

Please sign in to comment.