Skip to content

Commit

Permalink
cfq-iosched: reimplement priorities using different service trees
Browse files Browse the repository at this point in the history
We use different service trees for different priority classes.
This allows a simplification in the service tree insertion code, that no
longer has to consider priority while walking the tree.

Signed-off-by: Corrado Zoccolo <czoccolo@gmail.com>
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
  • Loading branch information
Corrado Zoccolo authored and Jens Axboe committed Oct 28, 2009
1 parent aa6f6a3 commit c0324a0
Showing 1 changed file with 82 additions and 34 deletions.
116 changes: 82 additions & 34 deletions block/cfq-iosched.c
Original file line number Diff line number Diff line change
Expand Up @@ -133,16 +133,32 @@ struct cfq_queue {
struct cfq_queue *new_cfqq;
};

/*
* Index in the service_trees.
* IDLE is handled separately, so it has negative index
*/
enum wl_prio_t {
IDLE_WORKLOAD = -1,
BE_WORKLOAD = 0,
RT_WORKLOAD = 1
};

/*
* Per block device queue structure
*/
struct cfq_data {
struct request_queue *queue;

/*
* rr list of queues with requests and the count of them
* rr lists of queues with requests, onle rr for each priority class.
* Counts are embedded in the cfq_rb_root
*/
struct cfq_rb_root service_trees[2];
struct cfq_rb_root service_tree_idle;
/*
* The priority currently being served
*/
struct cfq_rb_root service_tree;
enum wl_prio_t serving_prio;

/*
* Each priority tree is sorted by next_request position. These
Expand All @@ -152,7 +168,6 @@ struct cfq_data {
struct rb_root prio_trees[CFQ_PRIO_LISTS];

unsigned int busy_queues;
unsigned int busy_rt_queues;
unsigned int busy_queues_avg[2];

int rq_in_driver[2];
Expand Down Expand Up @@ -205,6 +220,15 @@ struct cfq_data {
unsigned long last_end_sync_rq;
};

static struct cfq_rb_root *service_tree_for(enum wl_prio_t prio,
struct cfq_data *cfqd)
{
if (prio == IDLE_WORKLOAD)
return &cfqd->service_tree_idle;

return &cfqd->service_trees[prio];
}

enum cfqq_state_flags {
CFQ_CFQQ_FLAG_on_rr = 0, /* on round-robin busy list */
CFQ_CFQQ_FLAG_wait_request, /* waiting for a request */
Expand Down Expand Up @@ -249,6 +273,23 @@ CFQ_CFQQ_FNS(coop);
#define cfq_log(cfqd, fmt, args...) \
blk_add_trace_msg((cfqd)->queue, "cfq " fmt, ##args)

static inline enum wl_prio_t cfqq_prio(struct cfq_queue *cfqq)
{
if (cfq_class_idle(cfqq))
return IDLE_WORKLOAD;
if (cfq_class_rt(cfqq))
return RT_WORKLOAD;
return BE_WORKLOAD;
}

static inline int cfq_busy_queues_wl(enum wl_prio_t wl, struct cfq_data *cfqd)
{
if (wl == IDLE_WORKLOAD)
return cfqd->service_tree_idle.count;

return cfqd->service_trees[wl].count;
}

static void cfq_dispatch_insert(struct request_queue *, struct request *);
static struct cfq_queue *cfq_get_queue(struct cfq_data *, bool,
struct io_context *, gfp_t);
Expand Down Expand Up @@ -332,10 +373,7 @@ cfq_get_avg_queues(struct cfq_data *cfqd, bool rt) {
unsigned min_q, max_q;
unsigned mult = cfq_hist_divisor - 1;
unsigned round = cfq_hist_divisor / 2;
unsigned busy = cfqd->busy_rt_queues;

if (!rt)
busy = cfqd->busy_queues - cfqd->busy_rt_queues;
unsigned busy = cfq_busy_queues_wl(rt, cfqd);

min_q = min(cfqd->busy_queues_avg[rt], busy);
max_q = max(cfqd->busy_queues_avg[rt], busy);
Expand Down Expand Up @@ -546,7 +584,7 @@ static unsigned long cfq_slice_offset(struct cfq_data *cfqd,
}

/*
* The cfqd->service_tree holds all pending cfq_queue's that have
* The cfqd->service_trees holds all pending cfq_queue's that have
* requests waiting to be processed. It is sorted in the order that
* we will service the queues.
*/
Expand All @@ -556,9 +594,10 @@ static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq,
struct rb_node **p, *parent;
struct cfq_queue *__cfqq;
unsigned long rb_key;
struct cfq_rb_root *service_tree = &cfqd->service_tree;
struct cfq_rb_root *service_tree;
int left;

service_tree = service_tree_for(cfqq_prio(cfqq), cfqd);
if (cfq_class_idle(cfqq)) {
rb_key = CFQ_IDLE_DELAY;
parent = rb_last(&service_tree->rb);
Expand Down Expand Up @@ -587,7 +626,8 @@ static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq,
/*
* same position, nothing more to do
*/
if (rb_key == cfqq->rb_key)
if (rb_key == cfqq->rb_key &&
cfqq->service_tree == service_tree)
return;

cfq_rb_erase(&cfqq->rb_node, cfqq->service_tree);
Expand All @@ -605,25 +645,14 @@ static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq,
__cfqq = rb_entry(parent, struct cfq_queue, rb_node);

/*
* sort RT queues first, we always want to give
* preference to them. IDLE queues goes to the back.
* after that, sort on the next service time.
* sort by key, that represents service time.
*/
if (cfq_class_rt(cfqq) > cfq_class_rt(__cfqq))
if (time_before(rb_key, __cfqq->rb_key))
n = &(*p)->rb_left;
else if (cfq_class_rt(cfqq) < cfq_class_rt(__cfqq))
n = &(*p)->rb_right;
else if (cfq_class_idle(cfqq) < cfq_class_idle(__cfqq))
n = &(*p)->rb_left;
else if (cfq_class_idle(cfqq) > cfq_class_idle(__cfqq))
n = &(*p)->rb_right;
else if (time_before(rb_key, __cfqq->rb_key))
n = &(*p)->rb_left;
else
else {
n = &(*p)->rb_right;

if (n == &(*p)->rb_right)
left = 0;
}

p = n;
}
Expand Down Expand Up @@ -722,8 +751,7 @@ static void cfq_add_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq)
BUG_ON(cfq_cfqq_on_rr(cfqq));
cfq_mark_cfqq_on_rr(cfqq);
cfqd->busy_queues++;
if (cfq_class_rt(cfqq))
cfqd->busy_rt_queues++;

cfq_resort_rr_list(cfqd, cfqq);
}

Expand All @@ -748,8 +776,6 @@ static void cfq_del_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq)

BUG_ON(!cfqd->busy_queues);
cfqd->busy_queues--;
if (cfq_class_rt(cfqq))
cfqd->busy_rt_queues--;
}

/*
Expand Down Expand Up @@ -1003,10 +1029,12 @@ static inline void cfq_slice_expired(struct cfq_data *cfqd, bool timed_out)
*/
static struct cfq_queue *cfq_get_next_queue(struct cfq_data *cfqd)
{
if (RB_EMPTY_ROOT(&cfqd->service_tree.rb))
return NULL;
struct cfq_rb_root *service_tree =
service_tree_for(cfqd->serving_prio, cfqd);

return cfq_rb_first(&cfqd->service_tree);
if (RB_EMPTY_ROOT(&service_tree->rb))
return NULL;
return cfq_rb_first(service_tree);
}

/*
Expand Down Expand Up @@ -1123,6 +1151,12 @@ static struct cfq_queue *cfq_close_cooperator(struct cfq_data *cfqd,
if (CFQQ_SEEKY(cfqq))
return NULL;

/*
* Do not merge queues of different priority classes
*/
if (cfq_class_rt(cfqq) != cfq_class_rt(cur_cfqq))
return NULL;

return cfqq;
}

Expand Down Expand Up @@ -1336,6 +1370,14 @@ static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd)
expire:
cfq_slice_expired(cfqd, 0);
new_queue:
if (!new_cfqq) {
if (cfq_busy_queues_wl(RT_WORKLOAD, cfqd))
cfqd->serving_prio = RT_WORKLOAD;
else if (cfq_busy_queues_wl(BE_WORKLOAD, cfqd))
cfqd->serving_prio = BE_WORKLOAD;
else
cfqd->serving_prio = IDLE_WORKLOAD;
}
cfqq = cfq_set_active_queue(cfqd, new_cfqq);
keep_queue:
return cfqq;
Expand All @@ -1362,8 +1404,12 @@ static int cfq_forced_dispatch(struct cfq_data *cfqd)
{
struct cfq_queue *cfqq;
int dispatched = 0;
int i;
for (i = 0; i < 2; ++i)
while ((cfqq = cfq_rb_first(&cfqd->service_trees[i])) != NULL)
dispatched += __cfq_forced_dispatch_cfqq(cfqq);

while ((cfqq = cfq_rb_first(&cfqd->service_tree)) != NULL)
while ((cfqq = cfq_rb_first(&cfqd->service_tree_idle)) != NULL)
dispatched += __cfq_forced_dispatch_cfqq(cfqq);

cfq_slice_expired(cfqd, 0);
Expand Down Expand Up @@ -2710,7 +2756,9 @@ static void *cfq_init_queue(struct request_queue *q)
if (!cfqd)
return NULL;

cfqd->service_tree = CFQ_RB_ROOT;
for (i = 0; i < 2; ++i)
cfqd->service_trees[i] = CFQ_RB_ROOT;
cfqd->service_tree_idle = CFQ_RB_ROOT;

/*
* Not strictly needed (since RB_ROOT just clears the node and we
Expand Down

0 comments on commit c0324a0

Please sign in to comment.