Skip to content

Commit

Permalink
n_tty: Fix unordered accesses to lockless read buffer
Browse files Browse the repository at this point in the history 
Add commit_head buffer index, which the producer-side publishes
after input processing in non-canon mode. This ensures the consumer-side
observes correctly-ordered writes in non-canonical mode (ie., the buffer
data is written before the buffer index is advanced). Fix consumer-side
uses of read_cnt() to use commit_head instead.

Add required memory barriers to the tail index to guarantee
the consumer-side has completed the loads before the producer-side
begins writing new data. Open-code the producer-side receive_room()
into the i/o loop.

Remove no-longer-referenced receive_room().

Based on work by Christian Riesch <christian.riesch@omicron.at>

Cc: Christian Riesch <christian.riesch@omicron.at>
Signed-off-by: Peter Hurley <peter@hurleysoftware.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
  • Loading branch information
Peter Hurley authored and Greg Kroah-Hartman committed Feb 2, 2015
1 parent 5e28cca commit 70aca71
Showing 1 changed file with 48 additions and 53 deletions.
101 changes: 48 additions & 53 deletions drivers/tty/n_tty.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@
struct n_tty_data {
/* producer-published */
size_t read_head;
size_t commit_head;
size_t canon_head;
size_t echo_head;
size_t echo_commit;
Expand Down Expand Up @@ -161,31 +162,6 @@ static inline int tty_put_user(struct tty_struct *tty, unsigned char x,
return put_user(x, ptr);
}

static int receive_room(struct tty_struct *tty)
{
struct n_tty_data *ldata = tty->disc_data;
int left;

if (I_PARMRK(tty)) {
/* Multiply read_cnt by 3, since each byte might take up to
* three times as many spaces when PARMRK is set (depending on
* its flags, e.g. parity error). */
left = N_TTY_BUF_SIZE - read_cnt(ldata) * 3 - 1;
} else
left = N_TTY_BUF_SIZE - read_cnt(ldata) - 1;

/*
* If we are doing input canonicalization, and there are no
* pending newlines, let characters through without limit, so
* that erase characters will be handled. Other excess
* characters will be beeped.
*/
if (left <= 0)
left = ldata->icanon && ldata->canon_head == ldata->read_tail;

return left;
}

/**
* n_tty_kick_worker - start input worker (if required)
* @tty: terminal
Expand Down Expand Up @@ -224,7 +200,7 @@ static ssize_t chars_in_buffer(struct tty_struct *tty)
ssize_t n = 0;

if (!ldata->icanon)
n = read_cnt(ldata);
n = ldata->commit_head - ldata->read_tail;
else
n = ldata->canon_head - ldata->read_tail;
return n;
Expand Down Expand Up @@ -318,10 +294,6 @@ static void n_tty_check_unthrottle(struct tty_struct *tty)
*
* n_tty_receive_buf()/producer path:
* caller holds non-exclusive termios_rwsem
* modifies read_head
*
* read_head is only considered 'published' if canonical mode is
* not active.
*/

static inline void put_tty_queue(unsigned char c, struct n_tty_data *ldata)
Expand All @@ -345,6 +317,7 @@ static void reset_buffer_flags(struct n_tty_data *ldata)
{
ldata->read_head = ldata->canon_head = ldata->read_tail = 0;
ldata->echo_head = ldata->echo_tail = ldata->echo_commit = 0;
ldata->commit_head = 0;
ldata->echo_mark = 0;
ldata->line_start = 0;

Expand Down Expand Up @@ -992,10 +965,6 @@ static inline void finish_erasing(struct n_tty_data *ldata)
*
* n_tty_receive_buf()/producer path:
* caller holds non-exclusive termios_rwsem
* modifies read_head
*
* Modifying the read_head is not considered a publish in this context
* because canonical mode is active -- only canon_head publishes
*/

static void eraser(unsigned char c, struct tty_struct *tty)
Expand Down Expand Up @@ -1144,7 +1113,6 @@ static void isig(int sig, struct tty_struct *tty)
*
* n_tty_receive_buf()/producer path:
* caller holds non-exclusive termios_rwsem
* publishes read_head via put_tty_queue()
*
* Note: may get exclusive termios_rwsem if flushing input buffer
*/
Expand Down Expand Up @@ -1214,7 +1182,6 @@ static void n_tty_receive_overrun(struct tty_struct *tty)
*
* n_tty_receive_buf()/producer path:
* caller holds non-exclusive termios_rwsem
* publishes read_head via put_tty_queue()
*/
static void n_tty_receive_parity_error(struct tty_struct *tty, unsigned char c)
{
Expand Down Expand Up @@ -1268,7 +1235,6 @@ n_tty_receive_signal_char(struct tty_struct *tty, int signal, unsigned char c)
* n_tty_receive_buf()/producer path:
* caller holds non-exclusive termios_rwsem
* publishes canon_head if canonical mode is active
* otherwise, publishes read_head via put_tty_queue()
*
* Returns 1 if LNEXT was received, else returns 0
*/
Expand Down Expand Up @@ -1381,7 +1347,7 @@ n_tty_receive_char_special(struct tty_struct *tty, unsigned char c)
handle_newline:
set_bit(ldata->read_head & (N_TTY_BUF_SIZE - 1), ldata->read_flags);
put_tty_queue(c, ldata);
ldata->canon_head = ldata->read_head;
smp_store_release(&ldata->canon_head, ldata->read_head);
kill_fasync(&tty->fasync, SIGIO, POLL_IN);
if (waitqueue_active(&tty->read_wait))
wake_up_interruptible_poll(&tty->read_wait, POLLIN);
Expand Down Expand Up @@ -1531,7 +1497,7 @@ n_tty_receive_char_lnext(struct tty_struct *tty, unsigned char c, char flag)
*
* n_tty_receive_buf()/producer path:
* claims non-exclusive termios_rwsem
* publishes read_head and canon_head
* publishes commit_head or canon_head
*/

static void
Expand All @@ -1542,16 +1508,14 @@ n_tty_receive_buf_real_raw(struct tty_struct *tty, const unsigned char *cp,
size_t n, head;

head = ldata->read_head & (N_TTY_BUF_SIZE - 1);
n = N_TTY_BUF_SIZE - max(read_cnt(ldata), head);
n = min_t(size_t, count, n);
n = min_t(size_t, count, N_TTY_BUF_SIZE - head);
memcpy(read_buf_addr(ldata, head), cp, n);
ldata->read_head += n;
cp += n;
count -= n;

head = ldata->read_head & (N_TTY_BUF_SIZE - 1);
n = N_TTY_BUF_SIZE - max(read_cnt(ldata), head);
n = min_t(size_t, count, n);
n = min_t(size_t, count, N_TTY_BUF_SIZE - head);
memcpy(read_buf_addr(ldata, head), cp, n);
ldata->read_head += n;
}
Expand Down Expand Up @@ -1681,8 +1645,13 @@ static void __receive_buf(struct tty_struct *tty, const unsigned char *cp,
tty->ops->flush_chars(tty);
}

if ((!ldata->icanon && (read_cnt(ldata) >= ldata->minimum_to_wake)) ||
L_EXTPROC(tty)) {
if (ldata->icanon && !L_EXTPROC(tty))
return;

/* publish read_head to consumer */
smp_store_release(&ldata->commit_head, ldata->read_head);

if ((read_cnt(ldata) >= ldata->minimum_to_wake) || L_EXTPROC(tty)) {
kill_fasync(&tty->fasync, SIGIO, POLL_IN);
if (waitqueue_active(&tty->read_wait))
wake_up_interruptible_poll(&tty->read_wait, POLLIN);
Expand All @@ -1699,7 +1668,31 @@ n_tty_receive_buf_common(struct tty_struct *tty, const unsigned char *cp,
down_read(&tty->termios_rwsem);

while (1) {
room = receive_room(tty);
/*
* When PARMRK is set, multiply read_cnt by 3, since each byte
* might take up to three times as many spaces (depending on
* its flags, e.g. parity error). [This calculation is wrong.]
*
* If we are doing input canonicalization, and there are no
* pending newlines, let characters through without limit, so
* that erase characters will be handled. Other excess
* characters will be beeped.
*
* paired with store in *_copy_from_read_buf() -- guarantees
* the consumer has loaded the data in read_buf up to the new
* read_tail (so this producer will not overwrite unread data)
*/
size_t tail = smp_load_acquire(&ldata->read_tail);
size_t head = ldata->read_head;

if (I_PARMRK(tty))
room = N_TTY_BUF_SIZE - (head - tail) * 3 - 1;
else
room = N_TTY_BUF_SIZE - (head - tail) - 1;

if (room <= 0)
room = ldata->icanon && ldata->canon_head == tail;

n = min(count, room);
if (!n) {
if (flow && !room)
Expand Down Expand Up @@ -1769,6 +1762,7 @@ static void n_tty_set_termios(struct tty_struct *tty, struct ktermios *old)
ldata->canon_head = ldata->read_head;
ldata->push = 1;
}
ldata->commit_head = ldata->read_head;
ldata->erasing = 0;
ldata->lnext = 0;
}
Expand Down Expand Up @@ -1909,7 +1903,7 @@ static inline int input_available_p(struct tty_struct *tty, int poll)
if (ldata->icanon && !L_EXTPROC(tty))
return ldata->canon_head != ldata->read_tail;
else
return read_cnt(ldata) >= amt;
return ldata->commit_head - ldata->read_tail >= amt;
}

/**
Expand Down Expand Up @@ -1941,20 +1935,22 @@ static int copy_from_read_buf(struct tty_struct *tty,
int retval;
size_t n;
bool is_eof;
size_t head = smp_load_acquire(&ldata->commit_head);
size_t tail = ldata->read_tail & (N_TTY_BUF_SIZE - 1);

retval = 0;
n = min(read_cnt(ldata), N_TTY_BUF_SIZE - tail);
n = min(head - ldata->read_tail, N_TTY_BUF_SIZE - tail);
n = min(*nr, n);
if (n) {
retval = copy_to_user(*b, read_buf_addr(ldata, tail), n);
n -= retval;
is_eof = n == 1 && read_buf(ldata, tail) == EOF_CHAR(tty);
tty_audit_add_data(tty, read_buf_addr(ldata, tail), n,
ldata->icanon);
ldata->read_tail += n;
smp_store_release(&ldata->read_tail, ldata->read_tail + n);
/* Turn single EOF into zero-length read */
if (L_EXTPROC(tty) && ldata->icanon && is_eof && !read_cnt(ldata))
if (L_EXTPROC(tty) && ldata->icanon && is_eof &&
(head == ldata->read_tail))
n = 0;
*b += n;
*nr -= n;
Expand Down Expand Up @@ -1997,7 +1993,7 @@ static int canon_copy_from_read_buf(struct tty_struct *tty,
bool eof_push = 0;

/* N.B. avoid overrun if nr == 0 */
n = min(*nr, read_cnt(ldata));
n = min(*nr, smp_load_acquire(&ldata->canon_head) - ldata->read_tail);
if (!n)
return 0;

Expand Down Expand Up @@ -2047,8 +2043,7 @@ static int canon_copy_from_read_buf(struct tty_struct *tty,

if (found)
clear_bit(eol, ldata->read_flags);
smp_mb__after_atomic();
ldata->read_tail += c;
smp_store_release(&ldata->read_tail, ldata->read_tail + c);

if (found) {
if (!ldata->push)
Expand Down

0 comments on commit 70aca71

Please sign in to comment.