Skip to content

Commit

Permalink
---
Browse files Browse the repository at this point in the history 
yaml
---
r: 26439
b: refs/heads/master
c: f6762b7
h: refs/heads/master
i:
  26437: 8e85c52
  26435: 1baecae
  26431: c05d126
v: v3
  • Loading branch information
Jens Axboe committed May 1, 2006
1 parent 39c035d commit a0d457a
Show file tree
Hide file tree
Showing 4 changed files with 127 additions and 31 deletions.
2 changes: 1 addition & 1 deletion [refs]
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
---
refs/heads/master: e27dedd84c119e2f7af54fcde3293be5ad812103
refs/heads/master: f6762b7ad8edd6abc802542ce845d3bc8adcb92f
141 changes: 118 additions & 23 deletions trunk/fs/pipe.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,8 @@ void pipe_wait(struct pipe_inode_info *pipe)
}

static int
pipe_iov_copy_from_user(void *to, struct iovec *iov, unsigned long len)
pipe_iov_copy_from_user(void *to, struct iovec *iov, unsigned long len,
int atomic)
{
unsigned long copy;

Expand All @@ -64,8 +65,13 @@ pipe_iov_copy_from_user(void *to, struct iovec *iov, unsigned long len)
iov++;
copy = min_t(unsigned long, len, iov->iov_len);

if (copy_from_user(to, iov->iov_base, copy))
return -EFAULT;
if (atomic) {
if (__copy_from_user_inatomic(to, iov->iov_base, copy))
return -EFAULT;
} else {
if (copy_from_user(to, iov->iov_base, copy))
return -EFAULT;
}
to += copy;
len -= copy;
iov->iov_base += copy;
Expand All @@ -75,7 +81,8 @@ pipe_iov_copy_from_user(void *to, struct iovec *iov, unsigned long len)
}

static int
pipe_iov_copy_to_user(struct iovec *iov, const void *from, unsigned long len)
pipe_iov_copy_to_user(struct iovec *iov, const void *from, unsigned long len,
int atomic)
{
unsigned long copy;

Expand All @@ -84,8 +91,13 @@ pipe_iov_copy_to_user(struct iovec *iov, const void *from, unsigned long len)
iov++;
copy = min_t(unsigned long, len, iov->iov_len);

if (copy_to_user(iov->iov_base, from, copy))
return -EFAULT;
if (atomic) {
if (__copy_to_user_inatomic(iov->iov_base, from, copy))
return -EFAULT;
} else {
if (copy_to_user(iov->iov_base, from, copy))
return -EFAULT;
}
from += copy;
len -= copy;
iov->iov_base += copy;
Expand All @@ -94,6 +106,47 @@ pipe_iov_copy_to_user(struct iovec *iov, const void *from, unsigned long len)
return 0;
}

/*
* Attempt to pre-fault in the user memory, so we can use atomic copies.
* Returns the number of bytes not faulted in.
*/
static int iov_fault_in_pages_write(struct iovec *iov, unsigned long len)
{
while (!iov->iov_len)
iov++;

while (len > 0) {
unsigned long this_len;

this_len = min_t(unsigned long, len, iov->iov_len);
if (fault_in_pages_writeable(iov->iov_base, this_len))
break;

len -= this_len;
iov++;
}

return len;
}

/*
* Pre-fault in the user memory, so we can use atomic copies.
*/
static void iov_fault_in_pages_read(struct iovec *iov, unsigned long len)
{
while (!iov->iov_len)
iov++;

while (len > 0) {
unsigned long this_len;

this_len = min_t(unsigned long, len, iov->iov_len);
fault_in_pages_readable(iov->iov_base, this_len);
len -= this_len;
iov++;
}
}

static void anon_pipe_buf_release(struct pipe_inode_info *pipe,
struct pipe_buffer *buf)
{
Expand All @@ -111,15 +164,24 @@ static void anon_pipe_buf_release(struct pipe_inode_info *pipe,
}

void *generic_pipe_buf_map(struct pipe_inode_info *pipe,
struct pipe_buffer *buf)
struct pipe_buffer *buf, int atomic)
{
if (atomic) {
buf->flags |= PIPE_BUF_FLAG_ATOMIC;
return kmap_atomic(buf->page, KM_USER0);
}

return kmap(buf->page);
}

void generic_pipe_buf_unmap(struct pipe_inode_info *pipe,
struct pipe_buffer *buf)
struct pipe_buffer *buf, void *map_data)
{
kunmap(buf->page);
if (buf->flags & PIPE_BUF_FLAG_ATOMIC) {
buf->flags &= ~PIPE_BUF_FLAG_ATOMIC;
kunmap_atomic(map_data, KM_USER0);
} else
kunmap(buf->page);
}

static int anon_pipe_buf_steal(struct pipe_inode_info *pipe,
Expand Down Expand Up @@ -183,7 +245,7 @@ pipe_readv(struct file *filp, const struct iovec *_iov,
struct pipe_buf_operations *ops = buf->ops;
void *addr;
size_t chars = buf->len;
int error;
int error, atomic;

if (chars > total_len)
chars = total_len;
Expand All @@ -195,12 +257,21 @@ pipe_readv(struct file *filp, const struct iovec *_iov,
break;
}

addr = ops->map(pipe, buf);
error = pipe_iov_copy_to_user(iov, addr + buf->offset, chars);
ops->unmap(pipe, buf);
atomic = !iov_fault_in_pages_write(iov, chars);
redo:
addr = ops->map(pipe, buf, atomic);
error = pipe_iov_copy_to_user(iov, addr + buf->offset, chars, atomic);
ops->unmap(pipe, buf, addr);
if (unlikely(error)) {
/*
* Just retry with the slow path if we failed.
*/
if (atomic) {
atomic = 0;
goto redo;
}
if (!ret)
ret = -EFAULT;
ret = error;
break;
}
ret += chars;
Expand Down Expand Up @@ -304,21 +375,28 @@ pipe_writev(struct file *filp, const struct iovec *_iov,
int offset = buf->offset + buf->len;

if (ops->can_merge && offset + chars <= PAGE_SIZE) {
int error, atomic = 1;
void *addr;
int error;

error = ops->pin(pipe, buf);
if (error)
goto out;

addr = ops->map(pipe, buf);
iov_fault_in_pages_read(iov, chars);
redo1:
addr = ops->map(pipe, buf, atomic);
error = pipe_iov_copy_from_user(offset + addr, iov,
chars);
ops->unmap(pipe, buf);
chars, atomic);
ops->unmap(pipe, buf, addr);
ret = error;
do_wakeup = 1;
if (error)
if (error) {
if (atomic) {
atomic = 0;
goto redo1;
}
goto out;
}
buf->len += chars;
total_len -= chars;
ret = chars;
Expand All @@ -341,7 +419,8 @@ pipe_writev(struct file *filp, const struct iovec *_iov,
int newbuf = (pipe->curbuf + bufs) & (PIPE_BUFFERS-1);
struct pipe_buffer *buf = pipe->bufs + newbuf;
struct page *page = pipe->tmp_page;
int error;
char *src;
int error, atomic = 1;

if (!page) {
page = alloc_page(GFP_HIGHUSER);
Expand All @@ -361,11 +440,27 @@ pipe_writev(struct file *filp, const struct iovec *_iov,
if (chars > total_len)
chars = total_len;

error = pipe_iov_copy_from_user(kmap(page), iov, chars);
kunmap(page);
iov_fault_in_pages_read(iov, chars);
redo2:
if (atomic)
src = kmap_atomic(page, KM_USER0);
else
src = kmap(page);

error = pipe_iov_copy_from_user(src, iov, chars,
atomic);
if (atomic)
kunmap_atomic(src, KM_USER0);
else
kunmap(page);

if (unlikely(error)) {
if (atomic) {
atomic = 0;
goto redo2;
}
if (!ret)
ret = -EFAULT;
ret = error;
break;
}
ret += chars;
Expand Down
4 changes: 2 additions & 2 deletions trunk/fs/splice.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -640,13 +640,13 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf,
/*
* Careful, ->map() uses KM_USER0!
*/
char *src = buf->ops->map(info, buf);
char *src = buf->ops->map(info, buf, 1);
char *dst = kmap_atomic(page, KM_USER1);

memcpy(dst + offset, src + buf->offset, this_len);
flush_dcache_page(page);
kunmap_atomic(dst, KM_USER1);
buf->ops->unmap(info, buf);
buf->ops->unmap(info, buf, src);
}

ret = mapping->a_ops->commit_write(file, page, offset, offset+this_len);
Expand Down
11 changes: 6 additions & 5 deletions trunk/include/linux/pipe_fs_i.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@

#define PIPE_BUFFERS (16)

#define PIPE_BUF_FLAG_LRU 0x01
#define PIPE_BUF_FLAG_LRU 0x01 /* page is on the LRU */
#define PIPE_BUF_FLAG_ATOMIC 0x02 /* was atomically mapped */

struct pipe_buffer {
struct page *page;
Expand All @@ -28,8 +29,8 @@ struct pipe_buffer {
*/
struct pipe_buf_operations {
int can_merge;
void * (*map)(struct pipe_inode_info *, struct pipe_buffer *);
void (*unmap)(struct pipe_inode_info *, struct pipe_buffer *);
void * (*map)(struct pipe_inode_info *, struct pipe_buffer *, int);
void (*unmap)(struct pipe_inode_info *, struct pipe_buffer *, void *);
int (*pin)(struct pipe_inode_info *, struct pipe_buffer *);
void (*release)(struct pipe_inode_info *, struct pipe_buffer *);
int (*steal)(struct pipe_inode_info *, struct pipe_buffer *);
Expand Down Expand Up @@ -64,8 +65,8 @@ void free_pipe_info(struct inode * inode);
void __free_pipe_info(struct pipe_inode_info *);

/* Generic pipe buffer ops functions */
void *generic_pipe_buf_map(struct pipe_inode_info *, struct pipe_buffer *);
void generic_pipe_buf_unmap(struct pipe_inode_info *, struct pipe_buffer *);
void *generic_pipe_buf_map(struct pipe_inode_info *, struct pipe_buffer *, int);
void generic_pipe_buf_unmap(struct pipe_inode_info *, struct pipe_buffer *, void *);
void generic_pipe_buf_get(struct pipe_inode_info *, struct pipe_buffer *);
int generic_pipe_buf_pin(struct pipe_inode_info *, struct pipe_buffer *);

Expand Down

0 comments on commit a0d457a

Please sign in to comment.